From 1cd38f7c962c3fbd7991815aa38834a9df4ec154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Fri, 7 Jul 2017 18:30:23 +0200 Subject: [PATCH 01/96] Add permissions configuration for file output --- libbeat/outputs/fileout/config.go | 1 + libbeat/outputs/fileout/file.go | 4 ++++ metricbeat/metricbeat.full.yml | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index 9e26a19cd21..26553f07d0e 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -13,6 +13,7 @@ type config struct { RotateEveryKb int `config:"rotate_every_kb" validate:"min=1"` NumberOfFiles int `config:"number_of_files"` Codec codec.Config `config:"codec"` + Permissions uint32 `config:"permissions"` } var ( diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index beaa5cb6211..d2fa538cc39 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -42,6 +42,9 @@ func (out *fileOutput) init(config config) error { out.rotator.Path = config.Path out.rotator.Name = config.Filename + if config.Permissions != 0 { + out.rotator.Permissions = &config.Permissions + } if out.rotator.Name == "" { out.rotator.Name = out.beat.Beat } @@ -53,6 +56,7 @@ func (out *fileOutput) init(config config) error { out.codec = enc + logp.Info("File output permissions set to: %v", *out.rotator.Permissions) logp.Info("File output path set to: %v", out.rotator.Path) logp.Info("File output base filename set to: %v", out.rotator.Name) diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 8073659c2b5..7d3aa597e8f 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -953,6 +953,10 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. + # default is 0600. + #permissions: 0666 + #----------------------------- Console output --------------------------------- #output.console: From e699a6f1ad038541d07316e8cd144e031728132f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Mon, 10 Jul 2017 10:25:21 +0200 Subject: [PATCH 02/96] Add permissions configuration for file output --- libbeat/docs/outputconfig.asciidoc | 5 +++++ libbeat/outputs/fileout/config.go | 1 + libbeat/outputs/fileout/file.go | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 6e0966f79e7..b2fa2cc28a9 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -958,6 +958,7 @@ output.file: filename: {beatname_lc} #rotate_every_kb: 10000 #number_of_files: 7 + #permissions: 0666 ------------------------------------------------------------------------------ ==== File Output Options @@ -993,6 +994,10 @@ The maximum number of files to save under <>. When this number of files is oldest file is deleted, and the rest of the files are shifted from last to first. The default is 7 files. +===== permissions + +Permissions to use for file creation. The default is 0600. + ===== codec Output codec configuration. If the `codec` section is missing, events will be json encoded. diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index 9e26a19cd21..26553f07d0e 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -13,6 +13,7 @@ type config struct { RotateEveryKb int `config:"rotate_every_kb" validate:"min=1"` NumberOfFiles int `config:"number_of_files"` Codec codec.Config `config:"codec"` + Permissions uint32 `config:"permissions"` } var ( diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index beaa5cb6211..d2fa538cc39 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -42,6 +42,9 @@ func (out *fileOutput) init(config config) error { out.rotator.Path = config.Path out.rotator.Name = config.Filename + if config.Permissions != 0 { + out.rotator.Permissions = &config.Permissions + } if out.rotator.Name == "" { out.rotator.Name = out.beat.Beat } @@ -53,6 +56,7 @@ func (out *fileOutput) init(config config) error { out.codec = enc + logp.Info("File output permissions set to: %v", *out.rotator.Permissions) logp.Info("File output path set to: %v", out.rotator.Path) logp.Info("File output base filename set to: %v", out.rotator.Name) From aca6d5ed4aadea9d79808319028f9d8f7274d903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Mon, 10 Jul 2017 10:26:59 +0200 Subject: [PATCH 03/96] Small change --- metricbeat/metricbeat.full.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 7d3aa597e8f..f69d515d46e 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -953,8 +953,7 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 - # Permissions to use for file creation. - # default is 0600. + # Permissions to use for file creation. The default is 0600. #permissions: 0666 From 2f407bd3dbd6fc4afa8bc076e8973a3dafa6b9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Wed, 19 Jul 2017 10:49:19 +0200 Subject: [PATCH 04/96] Change config.reference.yml appropriatly & fix wrong value in docs --- auditbeat/auditbeat.reference.yml | 3 +++ filebeat/filebeat.reference.yml | 3 +++ heartbeat/heartbeat.reference.yml | 3 +++ libbeat/_meta/config.reference.yml | 3 +++ libbeat/docs/outputconfig.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 2 +- packetbeat/packetbeat.reference.yml | 3 +++ winlogbeat/winlogbeat.reference.yml | 3 +++ 8 files changed, 20 insertions(+), 2 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 279d62d0bee..3db7d92a8d9 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -599,6 +599,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 92dd3487798..904993c5b43 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -971,6 +971,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 5c0c035a54e..552ea0c913b 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -749,6 +749,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 4a089c6f364..6f4afba1e56 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -535,6 +535,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index b2fa2cc28a9..246cec43465 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -958,7 +958,7 @@ output.file: filename: {beatname_lc} #rotate_every_kb: 10000 #number_of_files: 7 - #permissions: 0666 + #permissions: 0600 ------------------------------------------------------------------------------ ==== File Output Options diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index b23008b6481..abbc41f7f5f 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -947,7 +947,7 @@ output.elasticsearch: #number_of_files: 7 # Permissions to use for file creation. The default is 0600. - #permissions: 0666 + #permissions: 0600 #----------------------------- Console output --------------------------------- diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 34663929cdc..db6f24dcd7d 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -987,6 +987,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index dd6e8f237ab..9c3143cb571 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -564,6 +564,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: From e2c5acee882bb47998558b3da7fd7177e9b26106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Wed, 19 Jul 2017 11:29:02 +0200 Subject: [PATCH 05/96] Fix bug when no value given for permissions & add proper default value to config.go and file_rotator.go --- libbeat/logp/file_rotator.go | 15 +++++++-------- libbeat/outputs/fileout/config.go | 1 + libbeat/outputs/fileout/file.go | 7 +++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/libbeat/logp/file_rotator.go b/libbeat/logp/file_rotator.go index c7db4d7acd6..b528618d898 100644 --- a/libbeat/logp/file_rotator.go +++ b/libbeat/logp/file_rotator.go @@ -12,6 +12,7 @@ import ( const RotatorMaxFiles = 1024 const DefaultKeepFiles = 7 const DefaultRotateEveryBytes = 10 * 1024 * 1024 +const DefaultPermissions = 0600 type FileRotator struct { Path string @@ -56,6 +57,11 @@ func (rotator *FileRotator) CheckIfConfigSane() error { *rotator.RotateEveryBytes = DefaultRotateEveryBytes } + if rotator.Permissions == nil { + rotator.Permissions = new(uint32) + *rotator.Permissions = DefaultPermissions + } + if *rotator.KeepFiles < 2 || *rotator.KeepFiles >= RotatorMaxFiles { return fmt.Errorf("the number of files to keep should be between 2 and %d", RotatorMaxFiles-1) } @@ -164,7 +170,7 @@ func (rotator *FileRotator) Rotate() error { // create the new file path := rotator.FilePath(0) - current, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(rotator.getPermissions())) + current, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(*rotator.Permissions)) if err != nil { return err } @@ -177,10 +183,3 @@ func (rotator *FileRotator) Rotate() error { return nil } - -func (rotator *FileRotator) getPermissions() uint32 { - if rotator.Permissions == nil { - return 0600 - } - return *rotator.Permissions -} diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index 26553f07d0e..a970e2025e6 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -20,6 +20,7 @@ var ( defaultConfig = config{ NumberOfFiles: 7, RotateEveryKb: 10 * 1024, + Permissions: 0600, } ) diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index b1714179f87..59970c63e2d 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -47,9 +47,6 @@ func (out *fileOutput) init(config config) error { out.rotator.Path = config.Path out.rotator.Name = config.Filename - if config.Permissions != 0 { - out.rotator.Permissions = &config.Permissions - } if out.rotator.Name == "" { out.rotator.Name = out.beat.Beat } @@ -61,10 +58,12 @@ func (out *fileOutput) init(config config) error { out.codec = enc - logp.Info("File output permissions set to: %v", *out.rotator.Permissions) logp.Info("File output path set to: %v", out.rotator.Path) logp.Info("File output base filename set to: %v", out.rotator.Name) + logp.Info("File output permissions set to: %#o", config.Permissions) + out.rotator.Permissions = &config.Permissions + rotateeverybytes := uint64(config.RotateEveryKb) * 1024 logp.Info("Rotate every bytes set to: %v", rotateeverybytes) out.rotator.RotateEveryBytes = &rotateeverybytes From a49463ca0e83fa6671522a19978c599fd5e6fd02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 7 Sep 2017 21:49:49 +0200 Subject: [PATCH 06/96] Keep docker & k8s pod annotations while they are needed (#5084) In some cases pod annotations are neede after the pod is deleted, for instance when filebeat is reading the log behind the container. This change makes sure we keep metadata after a pod is gone. By storing access times we ensure that it's available as long as it's being used --- CHANGELOG.asciidoc | 2 + libbeat/docs/processors-using.asciidoc | 23 +- .../add_docker_metadata.go | 3 +- .../add_docker_metadata_test.go | 2 + .../processors/add_docker_metadata/config.go | 6 + .../processors/add_docker_metadata/watcher.go | 111 ++++++++- .../add_docker_metadata/watcher_test.go | 212 ++++++++++++++++++ .../add_kubernetes_metadata/config.go | 14 +- .../add_kubernetes_metadata/kubernetes.go | 2 +- .../add_kubernetes_metadata/podwatcher.go | 79 ++++++- 10 files changed, 420 insertions(+), 34 deletions(-) create mode 100644 libbeat/processors/add_docker_metadata/watcher_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 74a3665950c..f2efb4799f7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -35,6 +35,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add support for `initContainers` in `add_kubernetes_metadata` processor. {issue}4825[4825] - Fix the `/usr/bin/beatname` script to accept `-d "*"` as a parameter. {issue}5040[5040] - Combine `fields.yml` properties when they are defined in different sources. {issue}5075[5075] +- Keep Docker & Kubernetes pod metadata after container dies while they are needed by processors. {pull}5084[5084] - Fix `fields.yml` lookup when using `export template` with a custom `path.config` param. {issue}5089[5089] *Auditbeat* @@ -79,6 +80,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Don't start filebeat if external modules/prospectors config is wrong and reload is disabled {pull}5053[5053] - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085] - Changed the number of shards in the default configuration to 3. {issue}5095[5095] +- Remove error log from runnerfactory as error is returned by API. {pull}5085[5085] *Heartbeat* diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 7fc0c14dd0c..e655cd5e2d9 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -562,6 +562,19 @@ processors: lookup_fields: ["metricset.host"] ------------------------------------------------------------------------------- +The `add_kubernetes_metadata` processor has the following configuration settings: + +`in_cluster`:: (Optional) Use in cluster settings for Kubernetes client, `true` +by default. +`host`:: (Optional) In case `in_cluster` is false, use this host to connect to +Kubernetes API. +`kube_config`:: (Optional) Use given config file as configuration for Kubernetes +client. +`default_indexers.enabled`:: (Optional) Enable/Disable default pod indexers, in +case you want to specify your own. +`default_matchers.enabled`:: (Optional) Enable/Disable default pod matchers, in +case you want to specify your own. + [[add-docker-metadata]] === Add Docker metadata @@ -578,10 +591,10 @@ from Docker containers: processors: - add_docker_metadata: host: "unix:///var/run/docker.sock" - match_fields: ["system.process.cgroup.id"] - match_source: true - match_source_index: 4 - + #match_fields: ["system.process.cgroup.id"] + #match_source: true + #match_source_index: 4 + #cleanup_timeout: 60 # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: # certificate_authority: "/etc/pki/root/ca.pem" @@ -600,3 +613,5 @@ It has the following settings: `match_source_index`:: (Optional) Index in the source path split by / to look for container id. It defaults to 4 to match `/var/lib/docker/containers//*.log` +`cleanup_timeout`:: (Optional) Time of inactivity to consider we can clean and +forget metadata for a container, 60s by default. diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index a90046318e4..024630150f1 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -79,7 +79,8 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { if event.Fields["source"] != nil { event, err = d.sourceProcessor.Run(event) if err != nil { - return nil, err + logp.Debug("docker", "Error while extracting container ID from source path: %v", err) + return event, nil } } } diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index dea7da0c738..cebd43ee3af 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -191,6 +191,8 @@ func (m *mockWatcher) Start() error { return nil } +func (m *mockWatcher) Stop() {} + func (m *mockWatcher) Container(ID string) *Container { return m.containers[ID] } diff --git a/libbeat/processors/add_docker_metadata/config.go b/libbeat/processors/add_docker_metadata/config.go index 0efcef7e6ec..77fb02b49a5 100644 --- a/libbeat/processors/add_docker_metadata/config.go +++ b/libbeat/processors/add_docker_metadata/config.go @@ -1,5 +1,7 @@ package add_docker_metadata +import "time" + // Config for docker processor type Config struct { Host string `config:"host"` @@ -7,6 +9,10 @@ type Config struct { Fields []string `config:"match_fields"` MatchSource bool `config:"match_source"` SourceIndex int `config:"match_source_index"` + + // Annotations are kept after container is killled, until they haven't been accessed + // for a full `cleanup_timeout`: + CleanupTimeout time.Duration `config:"cleanup_timeout"` } // TLSConfig for docker socket connection diff --git a/libbeat/processors/add_docker_metadata/watcher.go b/libbeat/processors/add_docker_metadata/watcher.go index 607dec7f2ec..4cfee779bc9 100644 --- a/libbeat/processors/add_docker_metadata/watcher.go +++ b/libbeat/processors/add_docker_metadata/watcher.go @@ -1,15 +1,17 @@ package add_docker_metadata import ( - "context" "fmt" "net/http" + "sync" "time" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/events" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/docker/go-connections/tlsconfig" + "golang.org/x/net/context" "github.com/elastic/beats/libbeat/logp" ) @@ -22,6 +24,9 @@ type Watcher interface { // Start watching docker API for new containers Start() error + // Stop watching docker API for new containers + Stop() + // Container returns the running container with the given ID or nil if unknown Container(ID string) *Container @@ -30,11 +35,15 @@ type Watcher interface { } type watcher struct { - client *client.Client + sync.RWMutex + client Client ctx context.Context stop context.CancelFunc containers map[string]*Container + deleted map[string]time.Time // deleted annotations key -> last access time + cleanupTimeout time.Duration lastValidTimestamp int64 + stopped sync.WaitGroup } // Container info retrieved by the watcher @@ -45,6 +54,12 @@ type Container struct { Labels map[string]string } +// Client for docker interface +type Client interface { + ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) + Events(ctx context.Context, options types.EventsOptions) (<-chan events.Message, <-chan error) +} + type WatcherConstructor func(host string, tls *TLSConfig) (Watcher, error) // NewWatcher returns a watcher running for the given settings @@ -69,28 +84,51 @@ func NewWatcher(host string, tls *TLSConfig) (Watcher, error) { } } - cli, err := client.NewClient(host, dockerAPIVersion, httpClient, nil) + client, err := client.NewClient(host, dockerAPIVersion, httpClient, nil) if err != nil { return nil, err } + return NewWatcherWithClient(client, 60*time.Second) +} + +func NewWatcherWithClient(client Client, cleanupTimeout time.Duration) (*watcher, error) { ctx, cancel := context.WithCancel(context.Background()) return &watcher{ - client: cli, - ctx: ctx, - stop: cancel, - containers: make(map[string]*Container), + client: client, + ctx: ctx, + stop: cancel, + containers: make(map[string]*Container), + deleted: make(map[string]time.Time), + cleanupTimeout: cleanupTimeout, }, nil } // Container returns the running container with the given ID or nil if unknown func (w *watcher) Container(ID string) *Container { - return w.containers[ID] + w.RLock() + container := w.containers[ID] + w.RUnlock() + + // Update last access time if it's deleted + if _, ok := w.deleted[ID]; ok { + w.Lock() + w.deleted[ID] = time.Now() + w.Unlock() + } + + return container } // Containers returns the list of known containers func (w *watcher) Containers() map[string]*Container { - return w.containers + w.RLock() + defer w.RUnlock() + res := make(map[string]*Container) + for k, v := range w.containers { + res[k] = v + } + return res } // Start watching docker API for new containers @@ -99,6 +137,8 @@ func (w *watcher) Start() error { logp.Debug("docker", "Start docker containers scanner") w.lastValidTimestamp = time.Now().Unix() + w.Lock() + defer w.Unlock() containers, err := w.client.ContainerList(w.ctx, types.ContainerListOptions{}) if err != nil { return err @@ -113,11 +153,17 @@ func (w *watcher) Start() error { } } + w.stopped.Add(2) go w.watch() + go w.cleanupWorker() return nil } +func (w *watcher) Stop() { + w.stop() +} + func (w *watcher) watch() { filters := filters.NewArgs() filters.Add("type", "container") @@ -138,22 +184,30 @@ func (w *watcher) watch() { w.lastValidTimestamp = event.Time // Add / update - if event.Action == "create" || event.Action == "update" { + if event.Action == "start" || event.Action == "update" { name := event.Actor.Attributes["name"] image := event.Actor.Attributes["image"] delete(event.Actor.Attributes, "name") delete(event.Actor.Attributes, "image") + + w.Lock() w.containers[event.Actor.ID] = &Container{ ID: event.Actor.ID, Name: name, Image: image, Labels: event.Actor.Attributes, } + + // un-delete if it's flagged (in case of update or recreation) + delete(w.deleted, event.Actor.ID) + w.Unlock() } // Delete if event.Action == "die" || event.Action == "kill" { - delete(w.containers, event.Actor.ID) + w.Lock() + w.deleted[event.Actor.ID] = time.Now() + w.Unlock() } case err := <-errors: @@ -164,8 +218,43 @@ func (w *watcher) watch() { case <-w.ctx.Done(): logp.Debug("docker", "Watcher stopped") + w.stopped.Done() return } } } } + +// Clean up deleted containers after they are not used anymore +func (w *watcher) cleanupWorker() { + for { + // Wait a full period + time.Sleep(w.cleanupTimeout) + + select { + case <-w.ctx.Done(): + w.stopped.Done() + return + default: + // Check entries for timeout + var toDelete []string + timeout := time.Now().Add(-w.cleanupTimeout) + w.RLock() + for key, lastSeen := range w.deleted { + if lastSeen.Before(timeout) { + logp.Debug("docker", "Removing container %s after cool down timeout") + toDelete = append(toDelete, key) + } + } + w.RUnlock() + + // Delete timed out entries: + w.Lock() + for _, key := range toDelete { + delete(w.deleted, key) + delete(w.containers, key) + } + w.Unlock() + } + } +} diff --git a/libbeat/processors/add_docker_metadata/watcher_test.go b/libbeat/processors/add_docker_metadata/watcher_test.go new file mode 100644 index 00000000000..3508722649a --- /dev/null +++ b/libbeat/processors/add_docker_metadata/watcher_test.go @@ -0,0 +1,212 @@ +package add_docker_metadata + +import ( + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/events" + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" +) + +type MockClient struct { + // containers to return on ContainerList call + containers []types.Container + // event list to send on Events call + events []interface{} + + done chan interface{} +} + +func (m *MockClient) ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) { + return m.containers, nil +} + +func (m *MockClient) Events(ctx context.Context, options types.EventsOptions) (<-chan events.Message, <-chan error) { + eventsC := make(chan events.Message) + errorsC := make(chan error) + + go func() { + for _, event := range m.events { + switch e := event.(type) { + case events.Message: + eventsC <- e + case error: + errorsC <- e + } + } + close(m.done) + }() + + return eventsC, errorsC +} + +func TestWatcherInitialization(t *testing.T) { + watcher := runWatcher(t, true, + []types.Container{ + types.Container{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + types.Container{ + ID: "6ac6ee8df5d4", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{}, + }, + }, + nil) + + assert.Equal(t, watcher.Containers(), map[string]*Container{ + "0332dbd79e20": &Container{ + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "6ac6ee8df5d4": &Container{ + ID: "6ac6ee8df5d4", + Name: "other", + Image: "nginx", + Labels: map[string]string{}, + }, + }) +} + +func TestWatcherAddEvents(t *testing.T) { + watcher := runWatcher(t, true, + []types.Container{ + types.Container{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + }, + []interface{}{ + events.Message{ + Action: "start", + Actor: events.Actor{ + ID: "6ac6ee8df5d4", + Attributes: map[string]string{ + "name": "other", + "image": "nginx", + "label": "value", + }, + }, + }, + }, + ) + + assert.Equal(t, watcher.Containers(), map[string]*Container{ + "0332dbd79e20": &Container{ + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "6ac6ee8df5d4": &Container{ + ID: "6ac6ee8df5d4", + Name: "other", + Image: "nginx", + Labels: map[string]string{"label": "value"}, + }, + }) +} + +func TestWatcherUpdateEvent(t *testing.T) { + watcher := runWatcher(t, true, + []types.Container{ + types.Container{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + }, + }, + []interface{}{ + events.Message{ + Action: "update", + Actor: events.Actor{ + ID: "0332dbd79e20", + Attributes: map[string]string{ + "name": "containername", + "image": "busybox", + "label": "bar", + }, + }, + }, + }, + ) + + assert.Equal(t, watcher.Containers(), map[string]*Container{ + "0332dbd79e20": &Container{ + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + }, + }) + assert.Equal(t, len(watcher.deleted), 0) +} + +func TestWatcherKill(t *testing.T) { + watcher := runWatcher(t, false, + []types.Container{ + types.Container{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + }, + }, + []interface{}{ + events.Message{ + Action: "kill", + Actor: events.Actor{ + ID: "0332dbd79e20", + }, + }, + }, + ) + + // Check it doesn't get removed while we request meta for the container + for i := 0; i < 18; i++ { + watcher.Container("0332dbd79e20") + assert.Equal(t, len(watcher.Containers()), 1) + time.Sleep(50 * time.Millisecond) + } + + // Now it should get removed + time.Sleep(150 * time.Millisecond) + assert.Equal(t, len(watcher.Containers()), 0) +} + +func runWatcher(t *testing.T, kill bool, containers []types.Container, events []interface{}) *watcher { + client := &MockClient{ + containers: containers, + events: events, + done: make(chan interface{}), + } + + watcher, err := NewWatcherWithClient(client, 100*time.Millisecond) + if err != nil { + t.Fatal(err) + } + + err = watcher.Start() + if err != nil { + t.Fatal(err) + } + + <-client.done + if kill { + watcher.Stop() + watcher.stopped.Wait() + } + + return watcher +} diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index ec18bf58ca6..6f1c1f0b898 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -7,11 +7,14 @@ import ( ) type kubeAnnotatorConfig struct { - InCluster bool `config:"in_cluster"` - KubeConfig string `config:"kube_config"` - Host string `config:"host"` - Namespace string `config:"namespace"` - SyncPeriod time.Duration `config:"sync_period"` + InCluster bool `config:"in_cluster"` + KubeConfig string `config:"kube_config"` + Host string `config:"host"` + Namespace string `config:"namespace"` + SyncPeriod time.Duration `config:"sync_period"` + // Annotations are kept after pod is removed, until they haven't been accessed + // for a full `cleanup_timeout`: + CleanupTimeout time.Duration `config:"cleanup_timeout"` Indexers PluginConfig `config:"indexers"` Matchers PluginConfig `config:"matchers"` DefaultMatchers Enabled `config:"default_matchers"` @@ -31,6 +34,7 @@ func defaultKuberentesAnnotatorConfig() kubeAnnotatorConfig { return kubeAnnotatorConfig{ InCluster: true, SyncPeriod: 1 * time.Second, + CleanupTimeout: 60 * time.Second, Namespace: "kube-system", DefaultMatchers: Enabled{true}, DefaultIndexers: Enabled{true}, diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index c41b65582c6..3be48a46596 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -175,7 +175,7 @@ func newKubernetesAnnotator(cfg *common.Config) (processors.Processor, error) { logp.Debug("kubernetes", "Using host ", config.Host) logp.Debug("kubernetes", "Initializing watcher") if client != nil { - watcher := NewPodWatcher(client, &indexers, config.SyncPeriod, config.Host) + watcher := NewPodWatcher(client, &indexers, config.SyncPeriod, config.CleanupTimeout, config.Host) if watcher.Run() { return &kubernetesAnnotator{podWatcher: watcher, matchers: &matchers}, nil diff --git a/libbeat/processors/add_kubernetes_metadata/podwatcher.go b/libbeat/processors/add_kubernetes_metadata/podwatcher.go index 2d0ef0e2fca..db539a52b65 100644 --- a/libbeat/processors/add_kubernetes_metadata/podwatcher.go +++ b/libbeat/processors/add_kubernetes_metadata/podwatcher.go @@ -17,6 +17,7 @@ import ( type PodWatcher struct { kubeClient *k8s.Client syncPeriod time.Duration + cleanupTimeout time.Duration podQueue chan *corev1.Pod nodeFilter k8s.Option lastResourceVersion string @@ -27,21 +28,21 @@ type PodWatcher struct { } type annotationCache struct { - sync.Mutex + sync.RWMutex annotations map[string]common.MapStr - pods map[string]*Pod // pod uid -> Pod + pods map[string]*Pod // pod uid -> Pod + deleted map[string]time.Time // deleted annotations key -> last access time } -type NodeOption struct{} - // NewPodWatcher initializes the watcher client to provide a local state of // pods from the cluster (filtered to the given host) -func NewPodWatcher(kubeClient *k8s.Client, indexers *Indexers, syncPeriod time.Duration, host string) *PodWatcher { +func NewPodWatcher(kubeClient *k8s.Client, indexers *Indexers, syncPeriod, cleanupTimeout time.Duration, host string) *PodWatcher { ctx, cancel := context.WithCancel(context.Background()) return &PodWatcher{ kubeClient: kubeClient, indexers: indexers, syncPeriod: syncPeriod, + cleanupTimeout: cleanupTimeout, podQueue: make(chan *corev1.Pod, 10), nodeFilter: k8s.QueryParam("fieldSelector", "spec.nodeName="+host), lastResourceVersion: "0", @@ -50,6 +51,7 @@ func NewPodWatcher(kubeClient *k8s.Client, indexers *Indexers, syncPeriod time.D annotationCache: annotationCache{ annotations: make(map[string]common.MapStr), pods: make(map[string]*Pod), + deleted: make(map[string]time.Time), }, } } @@ -102,8 +104,9 @@ func (p *PodWatcher) watchPods() { } func (p *PodWatcher) Run() bool { - // Start pod processing worker: + // Start pod processing & annotations cleanup workers go p.worker() + go p.cleanupWorker() // Make sure that events don't flow into the annotator before informer is fully set up // Sync initial state: @@ -133,6 +136,9 @@ func (p *PodWatcher) onPodAdd(pod *Pod) { for _, m := range metadata { p.annotationCache.annotations[m.Index] = m.Data + + // un-delete if it's flagged (in case of update or recreation) + delete(p.annotationCache.deleted, m.Index) } } @@ -151,8 +157,10 @@ func (p *PodWatcher) onPodDelete(pod *Pod) { delete(p.annotationCache.pods, pod.Metadata.UID) + // Flag all annotations as deleted (they will be still available for a while) + now := time.Now() for _, index := range p.indexers.GetIndexes(pod) { - delete(p.annotationCache.annotations, index) + p.annotationCache.deleted[index] = now } } @@ -189,18 +197,65 @@ func (p *PodWatcher) worker() { } } +// Check annotations flagged as deleted for their last access time, fully delete +// the ones older than p.cleanupTimeout +func (p *PodWatcher) cleanupWorker() { + for { + // Wait a full period + time.Sleep(p.cleanupTimeout) + + select { + case <-p.ctx.Done(): + return + default: + // Check entries for timeout + var toDelete []string + timeout := time.Now().Add(-p.cleanupTimeout) + p.annotationCache.RLock() + for key, lastSeen := range p.annotationCache.deleted { + if lastSeen.Before(timeout) { + toDelete = append(toDelete, key) + } + } + p.annotationCache.RUnlock() + + // Delete timed out entries: + p.annotationCache.Lock() + for _, key := range toDelete { + delete(p.annotationCache.deleted, key) + delete(p.annotationCache.annotations, key) + } + p.annotationCache.Unlock() + } + } +} + func (p *PodWatcher) GetMetaData(arg string) common.MapStr { - p.annotationCache.Lock() - defer p.annotationCache.Unlock() - if meta, ok := p.annotationCache.annotations[arg]; ok { + p.annotationCache.RLock() + meta, ok := p.annotationCache.annotations[arg] + var deleted bool + if ok { + _, deleted = p.annotationCache.deleted[arg] + } + p.annotationCache.RUnlock() + + // Update deleted last access + if deleted { + p.annotationCache.Lock() + p.annotationCache.deleted[arg] = time.Now() + p.annotationCache.Unlock() + } + + if ok { return meta } + return nil } func (p *PodWatcher) GetPod(uid string) *Pod { - p.annotationCache.Lock() - defer p.annotationCache.Unlock() + p.annotationCache.RLock() + defer p.annotationCache.RUnlock() return p.annotationCache.pods[uid] } From c76f14f22b04226efa5043ec99f77ad08e57b9b7 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 8 Sep 2017 05:32:25 -0400 Subject: [PATCH 07/96] Support receiving events from audit multicast group (#5081) Update to go-libaudit v0.0.6 Closes #4850 --- NOTICE | 4 +- .../module/audit/kernel/_meta/docs.asciidoc | 25 +- auditbeat/module/audit/kernel/audit_linux.go | 112 +++++- .../module/audit/kernel/audit_linux_test.go | 91 ++++- auditbeat/module/audit/kernel/config.go | 12 +- .../module/audit/kernel/config_linux_test.go | 8 + metricbeat/mb/testing/modules.go | 2 +- .../elastic/go-libaudit/CHANGELOG.md | 6 + .../aucoalesce/mknormalize_data.go | 106 ------ .../github.com/elastic/go-libaudit/audit.go | 22 +- .../go-libaudit/auparse/defs_audit_arches.go | 104 ------ .../auparse/mk_audit_exit_codes.go | 226 ------------ .../go-libaudit/auparse/mk_audit_msg_types.go | 324 ------------------ .../github.com/elastic/go-libaudit/netlink.go | 6 +- vendor/vendor.json | 46 +-- 15 files changed, 281 insertions(+), 813 deletions(-) delete mode 100644 vendor/github.com/elastic/go-libaudit/aucoalesce/mknormalize_data.go delete mode 100644 vendor/github.com/elastic/go-libaudit/auparse/defs_audit_arches.go delete mode 100644 vendor/github.com/elastic/go-libaudit/auparse/mk_audit_exit_codes.go delete mode 100644 vendor/github.com/elastic/go-libaudit/auparse/mk_audit_msg_types.go diff --git a/NOTICE b/NOTICE index 28087a70ecb..1fb3fd3fec1 100644 --- a/NOTICE +++ b/NOTICE @@ -314,8 +314,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------- Dependency: github.com/elastic/go-libaudit -Version: v0.0.5 -Revision: 3a005d3d0bbcee26d60e3ab2f1890699746f4da6 +Version: v0.0.6 +Revision: df0d4981f3fce65ffd3d7411dfec3e03231b491c License type (autodetected): Apache License 2.0 ./vendor/github.com/elastic/go-libaudit/LICENSE: -------------------------------------------------------------------- diff --git a/auditbeat/module/audit/kernel/_meta/docs.asciidoc b/auditbeat/module/audit/kernel/_meta/docs.asciidoc index 3b4dd92283b..dce75d808f6 100644 --- a/auditbeat/module/audit/kernel/_meta/docs.asciidoc +++ b/auditbeat/module/audit/kernel/_meta/docs.asciidoc @@ -12,13 +12,6 @@ This metricset establishes a subscription to the kernel to receive the events as they occur. So unlike most other metricsets, the `period` configuration option is unused because it is not implemented using polling. -The Linux kernel only supports a single subscriber to the audit events so this -metricset cannot be used simultaneously with a service like `auditd`. `auditd` -should be disabled if this metricset is being used. If you wish to continue to -use `auditd` instead of this metricset to receive audit messages from the kernel -then consider using {filebeat}/filebeat-module-auditd.html[Filebeat] to collect -the daemon's log files. - The Linux Audit Framework can send multiple messages for a single auditable event. For example, a `rename` syscall causes the kernel to send eight separate messages. Each message describes a different aspect of the activity that is @@ -48,6 +41,24 @@ following example shows all configuration options with their default values. kernel.include_warnings: false ---- +*`kernel.socket_type`*:: This optional setting controls the type of +socket that {beatname_uc} uses to receive events from the kernel. The two +options are `unicast` and `multicast`. ++ +`unicast` should be used when {beatname_uc} is the primary userspace daemon for +receiving audit events and managing the rules. Only a single process can receive +audit events through the "unicast" connection so any other daemons should be +stopped (e.g. stop `auditd`). ++ +`multicast` can be used in kernel versions 3.16 and newer. By using `multicast` +{beatname_uc} will receive an audit event broadcast that is not exclusive to a +a single process. This is ideal for situations where `auditd` is running and +managing the rules. If `multicast` is specified, but the kernel version is less +than 3.16 {beatname_uc} will automatically revert to `unicast`. ++ +By default {beatname_uc} will use `multicast` if the kernel version is 3.16 or +newer and no rules have been defined. Otherwise `unicast` will be used. + *`kernel.resolve_ids`*:: This boolean setting enables the resolution of UIDs and GIDs to their associated names. The default value is true. diff --git a/auditbeat/module/audit/kernel/audit_linux.go b/auditbeat/module/audit/kernel/audit_linux.go index 45314fb4566..59c21de8cb3 100644 --- a/auditbeat/module/audit/kernel/audit_linux.go +++ b/auditbeat/module/audit/kernel/audit_linux.go @@ -2,7 +2,9 @@ package kernel import ( "os" + "strconv" "strings" + "syscall" "time" "github.com/pkg/errors" @@ -48,16 +50,17 @@ type MetricSet struct { // New constructs a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Experimental("The %v metricset is a beta feature", metricsetName) + cfgwarn.Beta("The %v metricset is a beta feature", metricsetName) config := defaultConfig if err := base.Module().UnpackConfig(&config); err != nil { return nil, errors.Wrap(err, "failed to unpack the audit.kernel config") } - debugf("%v the metricset is running as euid=%v", logPrefix, os.Geteuid()) + _, _, kernel, _ := kernelVersion() + debugf("%v the metricset is running as euid=%v on kernel=%v", logPrefix, os.Geteuid(), kernel) - client, err := libaudit.NewAuditClient(nil) + client, err := newAuditClient(&config) if err != nil { return nil, errors.Wrap(err, "failed to create audit.kernel client") } @@ -71,6 +74,37 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { }, nil } +func newAuditClient(c *Config) (*libaudit.AuditClient, error) { + hasMulticast := hasMulticastSupport() + + switch c.SocketType { + // Attempt to determine the optimal socket_type. + case "": + // Use multicast only when no rules are present. Specifying rules + // implies you want control over the audit framework so you should be + // using unicast. + if rules, _ := c.rules(); len(rules) == 0 && hasMulticast { + c.SocketType = "multicast" + logp.Info("%v kernel.socket_type=multicast will be used.", logPrefix) + } + case "multicast": + if !hasMulticast { + logp.Warn("%v kernel.socket_type is set to multicast "+ + "but based on the kernel version multicast audit subscriptions "+ + "are not supported. unicast will be used instead.", logPrefix) + c.SocketType = "unicast" + } + } + + switch c.SocketType { + case "multicast": + return libaudit.NewMulticastAuditClient(nil) + default: + c.SocketType = "unicast" + return libaudit.NewAuditClient(nil) + } +} + // Run initializes the audit client and receives audit messages from the // kernel until the reporter's done channel is closed. func (ms *MetricSet) Run(reporter mb.PushReporter) { @@ -115,8 +149,14 @@ func (ms *MetricSet) addRules(reporter mb.PushReporter) error { return nil } + client, err := libaudit.NewAuditClient(nil) + if err != nil { + return errors.Wrap(err, "failed to create audit client for adding rules") + } + defer client.Close() + // Delete existing rules. - n, err := ms.client.DeleteRules() + n, err := client.DeleteRules() if err != nil { return errors.Wrap(err, "failed to delete existing rules") } @@ -125,7 +165,7 @@ func (ms *MetricSet) addRules(reporter mb.PushReporter) error { // Add rules from config. var failCount int for _, rule := range rules { - if err = ms.client.AddRule(rule.data); err != nil { + if err = client.AddRule(rule.data); err != nil { // Treat rule add errors as warnings and continue. err = errors.Wrapf(err, "failed to add kernel rule '%v'", rule.flags) reporter.Error(err) @@ -139,6 +179,17 @@ func (ms *MetricSet) addRules(reporter mb.PushReporter) error { } func (ms *MetricSet) initClient() error { + if ms.config.SocketType == "multicast" { + // This request will fail with EPERM if this process does not have + // CAP_AUDIT_CONTROL, but we will ignore the response. The user will be + // required to ensure that auditing is enabled if the process is only + // given CAP_AUDIT_READ. + err := ms.client.SetEnabled(true, libaudit.NoWait) + return errors.Wrap(err, "failed to enable auditing in the kernel") + } + + // Unicast client initialization (requires CAP_AUDIT_CONTROL and that the + // process be in initial PID namespace). status, err := ms.client.GetStatus() if err != nil { return errors.Wrap(err, "failed to get audit status") @@ -348,3 +399,54 @@ func (s *stream) ReassemblyComplete(msgs []*auparse.AuditMessage) { func (s *stream) EventsLost(count int) { lostMetric.Inc() } + +func hasMulticastSupport() bool { + // Check the kernel version because 3.16+ should have multicast + // support. + major, minor, _, err := kernelVersion() + if err != nil { + // Assume not supported. + return false + } + + switch { + case major > 3, + major == 3 && minor >= 16: + return true + } + + return false +} + +func kernelVersion() (major, minor int, full string, err error) { + var uname syscall.Utsname + if err := syscall.Uname(&uname); err != nil { + return 0, 0, "", err + } + + data := make([]byte, len(uname.Release)) + for i, v := range uname.Release { + if v == 0 { + break + } + data[i] = byte(v) + } + + release := string(data) + parts := strings.SplitN(release, ".", 3) + if len(parts) < 2 { + return 0, 0, release, errors.Errorf("failed to parse uname release '%v'", release) + } + + major, err = strconv.Atoi(parts[0]) + if err != nil { + return 0, 0, release, errors.Wrapf(err, "failed to parse major version from '%v'", release) + } + + minor, err = strconv.Atoi(parts[1]) + if err != nil { + return 0, 0, release, errors.Wrapf(err, "failed to parse minor version from '%v'", release) + } + + return major, minor, release, nil +} diff --git a/auditbeat/module/audit/kernel/audit_linux_test.go b/auditbeat/module/audit/kernel/audit_linux_test.go index 7e3cc227dca..a34e151fa9e 100644 --- a/auditbeat/module/audit/kernel/audit_linux_test.go +++ b/auditbeat/module/audit/kernel/audit_linux_test.go @@ -1,24 +1,30 @@ package kernel import ( + "flag" + "fmt" + "os" + "os/exec" "testing" "time" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/go-libaudit" ) +// Specify the -audit flag when running these tests to interact with the real +// kernel instead of mocks. If running in Docker this requires being in the +// host PID namespace (--pid=host) and having CAP_AUDIT_CONTROL and +// CAP_AUDIT_WRITE (so use --privileged). +var audit = flag.Bool("audit", false, "interact with the real audit framework") + var userLoginMsg = `type=USER_LOGIN msg=audit(1492896301.818:19955): pid=12635 uid=0 auid=4294967295 ses=4294967295 msg='op=login acct=28696E76616C6964207573657229 exe="/usr/sbin/sshd" hostname=? addr=179.38.151.221 terminal=sshd res=failed'` func TestData(t *testing.T) { // Create a mock netlink client that provides the expected responses. mock := NewMock(). - // GetRules response with zero rules. Used by DeleteAll rules. - returnACK().returnDone(). - // AddRule response. - returnACK(). - // AddRule response. - returnACK(). // Get Status response for initClient returnACK().returnStatus(). // Send a single audit message from the kernel. @@ -47,9 +53,74 @@ func getConfig() map[string]interface{} { "module": "audit", "metricsets": []string{"kernel"}, "kernel.failure_mode": "log", - "kernel.audit_rules": ` - -w /etc/passwd -p wa -k auth - -a always,exit -F arch=b64 -S execve -k exec - `, + "kernel.socket_type": "unicast", + } +} + +func TestMulticastClient(t *testing.T) { + if !*audit { + t.Skip("-audit was not specified") + } + + if !hasMulticastSupport() { + t.Skip("no multicast support") + } + + c := map[string]interface{}{ + "module": "audit", + "metricsets": []string{"kernel"}, + "kernel.socket_type": "multicast", + "kernel.audit_rules": fmt.Sprintf(` + -a always,exit -F arch=b64 -F ppid=%d -S execve -k exec + `, os.Getpid()), + } + + // Any commands executed by this process will generate events due to the + // PPID filter we applied to the rule. + time.AfterFunc(time.Second, func() { exec.Command("cat", "/proc/self/status").Output() }) + + ms := mbtest.NewPushMetricSet(t, c) + events, errs := mbtest.RunPushMetricSet(5*time.Second, ms) + if len(errs) > 0 { + t.Fatalf("received errors: %+v", errs) + } + + // The number of events is non-deterministic so there is no validation. + t.Logf("received %d messages via multicast", len(events)) +} + +func TestUnicastClient(t *testing.T) { + if !*audit { + t.Skip("-audit was not specified") + } + + c := map[string]interface{}{ + "module": "audit", + "metricsets": []string{"kernel"}, + "kernel.socket_type": "unicast", + "kernel.audit_rules": fmt.Sprintf(` + -a always,exit -F arch=b64 -F ppid=%d -S execve -k exec + `, os.Getpid()), + } + + // Any commands executed by this process will generate events due to the + // PPID filter we applied to the rule. + time.AfterFunc(time.Second, func() { exec.Command("cat", "/proc/self/status").Output() }) + + ms := mbtest.NewPushMetricSet(t, c) + events, errs := mbtest.RunPushMetricSet(5*time.Second, ms) + if len(errs) > 0 { + t.Fatalf("received errors: %+v", errs) + } + + t.Log(events) + assert.Len(t, events, 1) +} + +func TestKernelVersion(t *testing.T) { + major, minor, full, err := kernelVersion() + if err != nil { + t.Fatal(err) } + t.Logf("major=%v, minor=%v, full=%v", major, minor, full) } diff --git a/auditbeat/module/audit/kernel/config.go b/auditbeat/module/audit/kernel/config.go index cd0d51153a1..885ab47d4b3 100644 --- a/auditbeat/module/audit/kernel/config.go +++ b/auditbeat/module/audit/kernel/config.go @@ -22,6 +22,7 @@ type Config struct { RawMessage bool `config:"kernel.include_raw_message"` // Include the list of raw audit messages in the event. Warnings bool `config:"kernel.include_warnings"` // Include warnings in the event (for dev/debug purposes only). RulesBlob string `config:"kernel.audit_rules"` // Audit rules. One rule per line. + SocketType string `config:"kernel.socket_type"` // Socket type to use with the kernel (unicast or multicast). // Tuning options (advanced, use with care) ReassemblerMaxInFlight uint32 `config:"kernel.reassembler.max_in_flight"` @@ -35,7 +36,7 @@ type auditRule struct { } // Validate validates the rules specified in the config. -func (c Config) Validate() error { +func (c *Config) Validate() error { var errs multierror.Errors _, err := c.rules() if err != nil { @@ -45,6 +46,15 @@ func (c Config) Validate() error { if err != nil { errs = append(errs, err) } + + c.SocketType = strings.ToLower(c.SocketType) + switch c.SocketType { + case "", "unicast", "multicast": + default: + errs = append(errs, errors.Errorf("invalid kernel.socket_type "+ + "'%v' (use unicast, multicast, or don't set a value)", c.SocketType)) + } + return errs.Err() } diff --git a/auditbeat/module/audit/kernel/config_linux_test.go b/auditbeat/module/audit/kernel/config_linux_test.go index 6840846167c..fda2a42c943 100644 --- a/auditbeat/module/audit/kernel/config_linux_test.go +++ b/auditbeat/module/audit/kernel/config_linux_test.go @@ -65,6 +65,14 @@ func TestConfigValidateFailureMode(t *testing.T) { t.Log(err) } +func TestConfigValidateConnectionType(t *testing.T) { + config := defaultConfig + config.SocketType = "Satellite" + err := config.Validate() + assert.Error(t, err) + t.Log(err) +} + func parseConfig(t testing.TB, yaml string) (Config, error) { c, err := common.NewConfigWithYAML([]byte(yaml), "") if err != nil { diff --git a/metricbeat/mb/testing/modules.go b/metricbeat/mb/testing/modules.go index 9209dcd9df9..d2d865e26f0 100644 --- a/metricbeat/mb/testing/modules.go +++ b/metricbeat/mb/testing/modules.go @@ -74,7 +74,7 @@ func newMetricSet(t testing.TB, config interface{}) mb.MetricSet { } m, metricsets, err := mb.NewModule(c, mb.Registry) if err != nil { - t.Fatal(err) + t.Fatal("failed to create new MetricSet", err) } if m == nil { t.Fatal("no module instantiated") diff --git a/vendor/github.com/elastic/go-libaudit/CHANGELOG.md b/vendor/github.com/elastic/go-libaudit/CHANGELOG.md index c679bda2bb5..e5909621ef0 100644 --- a/vendor/github.com/elastic/go-libaudit/CHANGELOG.md +++ b/vendor/github.com/elastic/go-libaudit/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [0.0.6] + +### Added + +- Add support for listening for audit messages using a multicast group. #9 + ## [0.0.5] ### Changed diff --git a/vendor/github.com/elastic/go-libaudit/aucoalesce/mknormalize_data.go b/vendor/github.com/elastic/go-libaudit/aucoalesce/mknormalize_data.go deleted file mode 100644 index 92cb9ddb718..00000000000 --- a/vendor/github.com/elastic/go-libaudit/aucoalesce/mknormalize_data.go +++ /dev/null @@ -1,106 +0,0 @@ -// +build ignore - -package main - -import ( - "bytes" - "encoding/base64" - "flag" - "fmt" - "go/format" - "io/ioutil" - "os" - "text/template" -) - -var tpl = template.Must(template.New("normalizations").Parse(` -// mknormalize_data.go -// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT - -// Copyright 2017 Elasticsearch Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package aucoalesce - -import ( - "encoding/base64" - "fmt" -) - -var assets map[string][]byte - -func asset(key string) ([]byte, error) { - if assets == nil { - assets = map[string][]byte{} - - var value []byte - {{- range $asset := .Assets }} - value, _ = base64.StdEncoding.DecodeString("{{ $asset.Data }}") - assets["{{ $asset.Name }}"] = value{{ end }} - } - - if value, found := assets[key]; found { - return value, nil - } - return nil, fmt.Errorf("asset not found for key=%v", key) -} -`)) - -type asset struct { - Name string - Data string -} - -var assets []asset - -func addAsset(key, file string) error { - data, err := ioutil.ReadFile(file) - if err != nil { - return err - } - - assets = append(assets, asset{ - Name: key, - Data: base64.StdEncoding.EncodeToString(data), - }) - return nil -} - -type templateVars struct { - Assets []asset -} - -func main() { - flag.Parse() - - args := flag.Args() - if len(args)%2 != 0 { - fmt.Fprintln(os.Stderr, "error: expected pairs of arguments (a key and a filename)") - os.Exit(1) - } - - for i := 0; i < len(args); i += 2 { - addAsset(args[i], args[i+1]) - } - - var buf bytes.Buffer - tpl.Execute(&buf, templateVars{ - Assets: assets, - }) - bs, err := format.Source(buf.Bytes()) - if err != nil { - panic(err) - } - os.Stdout.Write(bs) -} diff --git a/vendor/github.com/elastic/go-libaudit/audit.go b/vendor/github.com/elastic/go-libaudit/audit.go index db427324510..cb3dfbc6eb9 100644 --- a/vendor/github.com/elastic/go-libaudit/audit.go +++ b/vendor/github.com/elastic/go-libaudit/audit.go @@ -43,6 +43,12 @@ const ( AuditSet ) +// Netlink groups. +const ( + NetlinkGroupNone = iota // Group 0 not used + NetlinkGroupReadLog // "best effort" read only socket +) + // WaitMode is a flag to control the behavior of methods that abstract // asynchronous communication for the caller. type WaitMode uint8 @@ -72,13 +78,27 @@ type AuditClient struct { Netlink NetlinkSendReceiver } +// NewMulticastAuditClient creates a new AuditClient that binds to the multicast +// socket subscribes to the audit group. The process should have the +// CAP_AUDIT_READ capability to use this. This audit client should not be used +// for command and control. The resp parameter is optional. If provided resp +// will receive a copy of all data read from the netlink socket. This is useful +// for debugging purposes. +func NewMulticastAuditClient(resp io.Writer) (*AuditClient, error) { + return newAuditClient(NetlinkGroupReadLog, resp) +} + // NewAuditClient creates a new AuditClient. The resp parameter is optional. If // provided resp will receive a copy of all data read from the netlink socket. // This is useful for debugging purposes. func NewAuditClient(resp io.Writer) (*AuditClient, error) { + return newAuditClient(NetlinkGroupNone, resp) +} + +func newAuditClient(netlinkGroups uint32, resp io.Writer) (*AuditClient, error) { buf := make([]byte, syscall.NLMSG_HDRLEN+AuditMessageMaxLength) - netlink, err := NewNetlinkClient(syscall.NETLINK_AUDIT, buf, resp) + netlink, err := NewNetlinkClient(syscall.NETLINK_AUDIT, netlinkGroups, buf, resp) if err != nil { return nil, err } diff --git a/vendor/github.com/elastic/go-libaudit/auparse/defs_audit_arches.go b/vendor/github.com/elastic/go-libaudit/auparse/defs_audit_arches.go deleted file mode 100644 index 8863ea9340c..00000000000 --- a/vendor/github.com/elastic/go-libaudit/auparse/defs_audit_arches.go +++ /dev/null @@ -1,104 +0,0 @@ -// mk_audit_arches.pl -// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT - -// Copyright 2017 Elasticsearch Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build ignore - -package auparse - -/* -#include -*/ -import "C" - -import "fmt" - -// AuditArch represents a machine architecture (i.e. arm, ppc, x86_64). -type AuditArch uint32 - -// List of architectures constants used by then kernel. -const ( - AUDIT_ARCH_AARCH64 AuditArch = C.AUDIT_ARCH_AARCH64 - AUDIT_ARCH_ARM AuditArch = C.AUDIT_ARCH_ARM - AUDIT_ARCH_ARMEB AuditArch = C.AUDIT_ARCH_ARMEB - AUDIT_ARCH_CRIS AuditArch = C.AUDIT_ARCH_CRIS - AUDIT_ARCH_FRV AuditArch = C.AUDIT_ARCH_FRV - AUDIT_ARCH_I386 AuditArch = C.AUDIT_ARCH_I386 - AUDIT_ARCH_IA64 AuditArch = C.AUDIT_ARCH_IA64 - AUDIT_ARCH_M32R AuditArch = C.AUDIT_ARCH_M32R - AUDIT_ARCH_M68K AuditArch = C.AUDIT_ARCH_M68K - AUDIT_ARCH_MIPS AuditArch = C.AUDIT_ARCH_MIPS - AUDIT_ARCH_MIPS64 AuditArch = C.AUDIT_ARCH_MIPS64 - AUDIT_ARCH_MIPS64N32 AuditArch = C.AUDIT_ARCH_MIPS64N32 - AUDIT_ARCH_MIPSEL AuditArch = C.AUDIT_ARCH_MIPSEL - AUDIT_ARCH_MIPSEL64 AuditArch = C.AUDIT_ARCH_MIPSEL64 - AUDIT_ARCH_MIPSEL64N32 AuditArch = C.AUDIT_ARCH_MIPSEL64N32 - AUDIT_ARCH_PARISC AuditArch = C.AUDIT_ARCH_PARISC - AUDIT_ARCH_PARISC64 AuditArch = C.AUDIT_ARCH_PARISC64 - AUDIT_ARCH_PPC AuditArch = C.AUDIT_ARCH_PPC - AUDIT_ARCH_PPC64 AuditArch = C.AUDIT_ARCH_PPC64 - AUDIT_ARCH_PPC64LE AuditArch = C.AUDIT_ARCH_PPC64LE - AUDIT_ARCH_S390 AuditArch = C.AUDIT_ARCH_S390 - AUDIT_ARCH_S390X AuditArch = C.AUDIT_ARCH_S390X - AUDIT_ARCH_SH AuditArch = C.AUDIT_ARCH_SH - AUDIT_ARCH_SH64 AuditArch = C.AUDIT_ARCH_SH64 - AUDIT_ARCH_SHEL AuditArch = C.AUDIT_ARCH_SHEL - AUDIT_ARCH_SHEL64 AuditArch = C.AUDIT_ARCH_SHEL64 - AUDIT_ARCH_SPARC AuditArch = C.AUDIT_ARCH_SPARC - AUDIT_ARCH_SPARC64 AuditArch = C.AUDIT_ARCH_SPARC64 - AUDIT_ARCH_X86_64 AuditArch = C.AUDIT_ARCH_X86_64 -) - -var AuditArchNames = map[AuditArch]string{ - AUDIT_ARCH_AARCH64: "aarch64", - AUDIT_ARCH_ARM: "arm", - AUDIT_ARCH_ARMEB: "armeb", - AUDIT_ARCH_CRIS: "cris", - AUDIT_ARCH_FRV: "frv", - AUDIT_ARCH_I386: "i386", - AUDIT_ARCH_IA64: "ia64", - AUDIT_ARCH_M32R: "m32r", - AUDIT_ARCH_M68K: "m68k", - AUDIT_ARCH_MIPS: "mips", - AUDIT_ARCH_MIPS64: "mips64", - AUDIT_ARCH_MIPS64N32: "mips64n32", - AUDIT_ARCH_MIPSEL: "mipsel", - AUDIT_ARCH_MIPSEL64: "mipsel64", - AUDIT_ARCH_MIPSEL64N32: "mipsel64n32", - AUDIT_ARCH_PARISC: "parisc", - AUDIT_ARCH_PARISC64: "parisc64", - AUDIT_ARCH_PPC: "ppc", - AUDIT_ARCH_PPC64: "ppc64", - AUDIT_ARCH_PPC64LE: "ppc64le", - AUDIT_ARCH_S390: "s390", - AUDIT_ARCH_S390X: "s390x", - AUDIT_ARCH_SH: "sh", - AUDIT_ARCH_SH64: "sh64", - AUDIT_ARCH_SHEL: "shel", - AUDIT_ARCH_SHEL64: "shel64", - AUDIT_ARCH_SPARC: "sparc", - AUDIT_ARCH_SPARC64: "sparc64", - AUDIT_ARCH_X86_64: "x86_64", -} - -func (a AuditArch) String() string { - name, found := AuditArchNames[a] - if found { - return name - } - - return fmt.Sprintf("unknown[%x]", uint32(a)) -} diff --git a/vendor/github.com/elastic/go-libaudit/auparse/mk_audit_exit_codes.go b/vendor/github.com/elastic/go-libaudit/auparse/mk_audit_exit_codes.go deleted file mode 100644 index 57ad8f41f19..00000000000 --- a/vendor/github.com/elastic/go-libaudit/auparse/mk_audit_exit_codes.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2017 Elasticsearch Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build ignore - -package main - -import ( - "bufio" - "bytes" - "flag" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "regexp" - "sort" - "strconv" - "text/template" -) - -const includeErrno = ` -#include -` - -type ErrorNumber struct { - Name string - Value int -} - -// TemplateParams is the data used in evaluating the template. -type TemplateParams struct { - Command string - NameToNum []ErrorNumber - NumToName []ErrorNumber -} - -const header = `// go run {{.Command}}.go -// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT - -// Copyright 2017 Elasticsearch Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -` - -var headerTmpl = template.Must(template.New("header").Parse(header)) - -const godefsTemplate = ` -package auparse - -/* -#include -*/ -import "C" - -// AuditErrnoToNum contains a mapping of POSIX error names to errnos (error numbers). -var AuditErrnoToNum = map[string]int{ -{{- range $err := .NameToNum }} - "{{ $err.Name }}": C.{{ $err.Name }}, -{{- end }} -} - -// AuditErrnoToName contains a mapping of errnos (error numbers) to POSIX names. -var AuditErrnoToName = map[int]string{ -{{- range $err := .NumToName }} - {{ $err.Value }}: "{{ $err.Name }}", -{{- end }} -} -` - -var tmpl = template.Must(template.New("message_types").Parse(godefsTemplate)) - -var ( - errnoDefRegex = regexp.MustCompile(`^#define\s+(E\w+)\s+(\w+)`) -) - -func readErrorNumbers() ([]ErrorNumber, error) { - cmd := exec.Command("gcc", "-E", "-dD", "-") - cmd.Stdin = bytes.NewBufferString(includeErrno) - out, err := cmd.Output() - if err != nil { - return nil, err - } - - errorToNum := map[string]int{} - s := bufio.NewScanner(bytes.NewReader(out)) - for s.Scan() { - matches := errnoDefRegex.FindStringSubmatch(s.Text()) - if len(matches) != 3 { - continue - } - errno, err := strconv.Atoi(matches[2]) - if err != nil { - errorToNum[matches[1]] = -1 - continue - } - - errorToNum[matches[1]] = errno - } - - var errnos []ErrorNumber - for name, value := range errorToNum { - errnos = append(errnos, ErrorNumber{ - Name: name, - Value: value, - }) - } - - sort.Slice(errnos, func(i, j int) bool { - return errnos[i].Value < errnos[j].Value - }) - - return errnos, nil -} - -func run() error { - tmp, err := ioutil.TempDir("", "mk_audit_exit_codes") - if err != nil { - return err - } - defer os.RemoveAll(tmp) - - if err := os.Chdir(tmp); err != nil { - return err - } - - errnos, err := readErrorNumbers() - if err != nil { - return err - } - - // Filter duplicates and sort by name. - var numToName []ErrorNumber - for _, errno := range errnos { - if errno.Value >= 0 { - numToName = append(numToName, errno) - } - } - - // Create output file. - f, err := os.Create("defs.go") - if err != nil { - return err - } - defer f.Close() - - // Evaluate template. - r := TemplateParams{ - Command: filepath.Base(os.Args[0]), - NameToNum: errnos, - NumToName: numToName, - } - if err := tmpl.Execute(f, r); err != nil { - return err - } - - output, err := exec.Command("go", "tool", "cgo", "-godefs", "defs.go").Output() - if err != nil { - return err - } - - buf := new(bytes.Buffer) - if err = headerTmpl.Execute(buf, r); err != nil { - return nil - } - - s := bufio.NewScanner(bytes.NewReader(output)) - for s.Scan() { - if !bytes.HasPrefix(s.Bytes(), []byte("//")) { - buf.Write(s.Bytes()) - buf.WriteByte('\n') - } - } - - if err = ioutil.WriteFile(flagOut, buf.Bytes(), 0644); err != nil { - return err - } - - _, err = exec.Command("gofmt", "-w", "-s", flagOut).Output() - if err != nil { - return err - } - - return nil -} - -var flagOut string - -func main() { - flag.StringVar(&flagOut, "out", "zaudit_exit_codes.go", "output file") - flag.Parse() - - var err error - flagOut, err = filepath.Abs(flagOut) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } -} diff --git a/vendor/github.com/elastic/go-libaudit/auparse/mk_audit_msg_types.go b/vendor/github.com/elastic/go-libaudit/auparse/mk_audit_msg_types.go deleted file mode 100644 index 340bba2949c..00000000000 --- a/vendor/github.com/elastic/go-libaudit/auparse/mk_audit_msg_types.go +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2017 Elasticsearch Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build ignore - -package main - -import ( - "bufio" - "bytes" - "flag" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "os/exec" - "path/filepath" - "regexp" - "strconv" - "text/template" -) - -// Min and max record/message numbers. -const ( - minRecordNum = 1000 - maxRecordNum = 3000 -) - -// TemplateParams is the data used in evaluating the template. -type TemplateParams struct { - Command string - RecordTypes map[int]string - RecordNames map[string]string -} - -const fileTemplate = ` -// go run {{.Command}}.go -// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT - -// Copyright 2017 Elasticsearch Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package auparse - -import ( - "errors" - "fmt" - "strconv" - "strings" -) - -var errInvalidAuditMessageTypName = errors.New("invalid message type") - -// AuditMessageType represents an audit message type used by the kernel. -type AuditMessageType uint16 - -// List of AuditMessageTypes. -const( -{{- range $recordNum, $recordType := .RecordTypes }} - {{ $recordType}} AuditMessageType = {{ $recordNum }} -{{- end }} -) - -var auditMessageTypeToName = map[AuditMessageType]string{ -{{- range $recordType, $recordName := .RecordNames }} - {{ $recordType }}: "{{ $recordName }}", -{{- end }} -} - -func (t AuditMessageType) String() string { - name, found := auditMessageTypeToName[t] - if found { - return name - } - - return fmt.Sprintf("UNKNOWN[%d]", uint16(t)) -} - -func (t AuditMessageType) MarshalText() (text []byte, err error) { - return []byte(strings.ToLower(t.String())), nil -} - -var auditMessageNameToType = map[string]AuditMessageType{ -{{- range $recordType, $recordName := .RecordNames }} - "{{ $recordName }}": {{ $recordType }}, -{{- end }} -} - -// GetAuditMessageType accepts a type name and returns its numerical -// representation. If the name is unknown and error is returned. -func GetAuditMessageType(name string) (AuditMessageType, error) { - typ, found := auditMessageNameToType[name] - if found { - return typ, nil - } - - // Parse type from UNKNOWN[1329]. - start := strings.IndexByte(name, '[') - if start == -1 { - return 0, errInvalidAuditMessageTypName - } - name = name[start+1:] - - end := strings.IndexByte(name, ']') - if end == -1 { - return 0, errInvalidAuditMessageTypName - } - name = name[:end] - - num, err := strconv.ParseUint(name, 10, 16) - if err != nil { - return 0, errInvalidAuditMessageTypName - } - - return AuditMessageType(num), nil -} -` - -var tmpl = template.Must(template.New("message_types").Parse(fileTemplate)) - -var ( - headers = []string{ - `https://raw.githubusercontent.com/linux-audit/audit-kernel/v4.7/include/uapi/linux/audit.h`, - `https://raw.githubusercontent.com/linux-audit/audit-userspace/990aa27ccd02f9743c4f4049887ab89678ab362a/lib/libaudit.h`, - `https://raw.githubusercontent.com/linux-audit/audit-userspace/990aa27ccd02f9743c4f4049887ab89678ab362a/lib/msg_typetab.h`, - } -) - -func DownloadFile(url, destinationDir string) (string, error) { - resp, err := http.Get(url) - if err != nil { - return "", fmt.Errorf("http get failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("download failed with http status %v", resp.StatusCode) - } - - name := filepath.Join(destinationDir, filepath.Base(url)) - f, err := os.Create(name) - if err != nil { - return "", fmt.Errorf("failed to create output file: %v", err) - } - - _, err = io.Copy(f, resp.Body) - if err != nil { - return "", fmt.Errorf("failed to write file to disk: %v", err) - } - - return name, nil -} - -var ( - // nameMappingRegex is used to parse name mappings from msg_typetab.h. - nameMappingRegex = regexp.MustCompile(`^_S\((AUDIT_\w+),\s+"(\w+)"`) - - // recordTypeDefinitionRegex is used to parse type definitions from audit - // header files. - recordTypeDefinitionRegex = regexp.MustCompile(`^#define\s+(AUDIT_\w+)\s+(\d+)`) - - // nameExtractorRegex extracts a name from constants that did not have - // an explicit name mapping in msg_typetab.h. - nameExtractorRegex = regexp.MustCompile(`^AUDIT_(\w+)`) -) - -func readMessageTypeTable() (map[string]string, error) { - f, err := os.Open("msg_typetab.h") - if err != nil { - return nil, err - } - defer f.Close() - - constantToStringName := map[string]string{} - s := bufio.NewScanner(f) - for s.Scan() { - matches := nameMappingRegex.FindStringSubmatch(s.Text()) - if len(matches) == 3 { - constantToStringName[matches[1]] = matches[2] - } - } - - return constantToStringName, nil -} - -func readRecordTypes() (map[string]int, error) { - out, err := exec.Command("gcc", "-E", "-dD", "libaudit.h", "audit.h").Output() - if err != nil { - return nil, err - } - - recordTypeToNum := map[string]int{} - s := bufio.NewScanner(bytes.NewReader(out)) - for s.Scan() { - matches := recordTypeDefinitionRegex.FindStringSubmatch(s.Text()) - if len(matches) != 3 { - continue - } - recordNum, _ := strconv.Atoi(matches[2]) - - // Filter constants. - if recordNum >= minRecordNum && recordNum <= maxRecordNum { - recordTypeToNum[matches[1]] = recordNum - } - } - - return recordTypeToNum, nil -} - -func run() error { - tmp, err := ioutil.TempDir("", "mk_audit_msg_types") - if err != nil { - return err - } - defer os.RemoveAll(tmp) - - // Download header files from the Linux audit project. - var files []string - for _, url := range headers { - f, err := DownloadFile(url, tmp) - if err != nil { - return err - } - files = append(files, f) - } - - if err := os.Chdir(tmp); err != nil { - return err - } - - recordTypeToStringName, err := readMessageTypeTable() - if err != nil { - return err - } - - recordTypeToNum, err := readRecordTypes() - if err != nil { - return err - } - - numToRecordType := map[int]string{} - for recordType := range recordTypeToStringName { - num, found := recordTypeToNum[recordType] - if !found { - return fmt.Errorf("missing definition of %v", recordType) - } - numToRecordType[num] = recordType - } - - for recordType, num := range recordTypeToNum { - // Do not replace existing mappings. - if _, found := numToRecordType[num]; found { - continue - } - - numToRecordType[num] = recordType - - matches := nameExtractorRegex.FindStringSubmatch(recordType) - if len(matches) == 2 { - recordTypeToStringName[recordType] = matches[1] - } - } - - // Create output file. - f, err := os.Create(flagOut) - if err != nil { - return err - } - defer f.Close() - - // Evaluate template. - r := TemplateParams{ - Command: filepath.Base(os.Args[0]), - RecordTypes: numToRecordType, - RecordNames: recordTypeToStringName, - } - if err := tmpl.Execute(f, r); err != nil { - return err - } - - return nil -} - -var flagOut string - -func main() { - flag.StringVar(&flagOut, "out", "zaudit_msg_types.go", "output file") - flag.Parse() - - var err error - flagOut, err = filepath.Abs(flagOut) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } -} diff --git a/vendor/github.com/elastic/go-libaudit/netlink.go b/vendor/github.com/elastic/go-libaudit/netlink.go index aa111d8d4cd..1300245b25a 100644 --- a/vendor/github.com/elastic/go-libaudit/netlink.go +++ b/vendor/github.com/elastic/go-libaudit/netlink.go @@ -73,16 +73,16 @@ type NetlinkClient struct { // (this is useful for debugging). // // The returned NetlinkClient must be closed with Close() when finished. -func NewNetlinkClient(proto int, readBuf []byte, resp io.Writer) (*NetlinkClient, error) { +func NewNetlinkClient(proto int, groups uint32, readBuf []byte, resp io.Writer) (*NetlinkClient, error) { s, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, proto) if err != nil { return nil, err } - src := &syscall.SockaddrNetlink{Family: syscall.AF_NETLINK} + src := &syscall.SockaddrNetlink{Family: syscall.AF_NETLINK, Groups: groups} if err = syscall.Bind(s, src); err != nil { syscall.Close(s) - return nil, err + return nil, errors.Wrap(err, "bind failed") } pid, err := getPortID(s) diff --git a/vendor/vendor.json b/vendor/vendor.json index 69708a13963..22dc49a4a90 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -359,44 +359,44 @@ "revisionTime": "2016-08-05T00:47:13Z" }, { - "checksumSHA1": "Pd9e8K98V1LGGScIG/u4Z36l0P0=", + "checksumSHA1": "/JppfHtRGQSFjB9FlY2jpHhmlhk=", "path": "github.com/elastic/go-libaudit", - "revision": "3a005d3d0bbcee26d60e3ab2f1890699746f4da6", - "revisionTime": "2017-07-24T13:10:22Z", - "version": "v0.0.5", - "versionExact": "v0.0.5" + "revision": "df0d4981f3fce65ffd3d7411dfec3e03231b491c", + "revisionTime": "2017-09-07T20:19:58Z", + "version": "v0.0.6", + "versionExact": "v0.0.6" }, { - "checksumSHA1": "Ca2OhodWbbke1v+cctf9y/HdwZM=", + "checksumSHA1": "n8bRlhOdmfREBoCgStzHWGWiwSY=", "path": "github.com/elastic/go-libaudit/aucoalesce", - "revision": "3a005d3d0bbcee26d60e3ab2f1890699746f4da6", - "revisionTime": "2017-07-24T13:10:22Z", - "version": "v0.0.5", - "versionExact": "v0.0.5" + "revision": "df0d4981f3fce65ffd3d7411dfec3e03231b491c", + "revisionTime": "2017-09-07T20:19:58Z", + "version": "v0.0.6", + "versionExact": "v0.0.6" }, { - "checksumSHA1": "N+9Ssj7RWKl2Sk1DqVmZ2Z3f8Ts=", + "checksumSHA1": "eUIiDm0pSFKNKjWme5s3PtWEoSU=", "path": "github.com/elastic/go-libaudit/auparse", - "revision": "3a005d3d0bbcee26d60e3ab2f1890699746f4da6", - "revisionTime": "2017-07-24T13:10:22Z", - "version": "v0.0.5", - "versionExact": "v0.0.5" + "revision": "df0d4981f3fce65ffd3d7411dfec3e03231b491c", + "revisionTime": "2017-09-07T20:19:58Z", + "version": "v0.0.6", + "versionExact": "v0.0.6" }, { "checksumSHA1": "H0rnscnKHbkjmXc4whC3gtIPR0c=", "path": "github.com/elastic/go-libaudit/rule", - "revision": "3a005d3d0bbcee26d60e3ab2f1890699746f4da6", - "revisionTime": "2017-07-24T13:10:22Z", - "version": "v0.0.5", - "versionExact": "v0.0.5" + "revision": "df0d4981f3fce65ffd3d7411dfec3e03231b491c", + "revisionTime": "2017-09-07T20:19:58Z", + "version": "v0.0.6", + "versionExact": "v0.0.6" }, { "checksumSHA1": "36UaYid29Kyhrsa5D8N6BoM8dVw=", "path": "github.com/elastic/go-libaudit/rule/flags", - "revision": "3a005d3d0bbcee26d60e3ab2f1890699746f4da6", - "revisionTime": "2017-07-24T13:10:22Z", - "version": "v0.0.5", - "versionExact": "v0.0.5" + "revision": "df0d4981f3fce65ffd3d7411dfec3e03231b491c", + "revisionTime": "2017-09-07T20:19:58Z", + "version": "v0.0.6", + "versionExact": "v0.0.6" }, { "checksumSHA1": "3jizmlZPCyo6FAZY8Trk9jA8NH4=", From ac9561da99265cb85551f98b37990326d4104bcc Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 8 Sep 2017 05:56:07 -0400 Subject: [PATCH 08/96] Remove alias from perfmon docs (#5130) `alias` was removed in #4502. --- metricbeat/module/windows/perfmon/_meta/docs.asciidoc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc index d5297645ec3..68fcee1c362 100644 --- a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc +++ b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc @@ -22,13 +22,11 @@ For wildcard queries this setting has no effect. metricsets: ["perfmon"] period: 10s perfmon.counters: - - alias: "procesor.name.stats" - instance_label: "processor.name" + - instance_label: "processor.name" instance_name: "Total" measurement_label: "processor.time.total.pct" query: '\Processor Information(_Total)\% Processor Time' - - alias: "diskio.name.stats" - instance_label: "diskio.name" + - instance_label: "diskio.name" measurement_label: "diskio.write.bytes" query: '\PhysicalDisk(*)\Disk Writes/sec' format: "long" From 01841ce143164c85ad36b3674721309891c763a6 Mon Sep 17 00:00:00 2001 From: Wang Guoliang Date: Fri, 8 Sep 2017 17:57:42 +0800 Subject: [PATCH 09/96] should use strings.Contains(string(cmdline), process) instead (#5102) --- packetbeat/procs/procs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packetbeat/procs/procs.go b/packetbeat/procs/procs.go index 22226508aeb..5054354e6a0 100644 --- a/packetbeat/procs/procs.go +++ b/packetbeat/procs/procs.go @@ -173,7 +173,7 @@ func findPidsByCmdlineGrep(prefix string, process string) ([]int, error) { continue } - if strings.Index(string(cmdline), process) >= 0 { + if strings.Contains(string(cmdline), process) { pids = append(pids, pid) } } From 9f2795eb1ca7728a2a191f4363d629a58ea0f886 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Fri, 8 Sep 2017 11:58:56 +0200 Subject: [PATCH 10/96] Add missing link texts to fields references (#4919) --- heartbeat/docs/heartbeat-options.asciidoc | 2 +- packetbeat/docs/packetbeat-options.asciidoc | 4 ++-- winlogbeat/docs/winlogbeat-options.asciidoc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/heartbeat/docs/heartbeat-options.asciidoc b/heartbeat/docs/heartbeat-options.asciidoc index afd22133914..6c98d7ef759 100644 --- a/heartbeat/docs/heartbeat-options.asciidoc +++ b/heartbeat/docs/heartbeat-options.asciidoc @@ -139,7 +139,7 @@ will be overwritten by the value declared here. [[monitor-fields-under-root]] ==== `fields_under_root` -If this option is set to true, the custom <> +If this option is set to true, the custom <> are stored as top-level fields in the output document instead of being grouped under a `fields` sub-dictionary. If the custom field names conflict with other field names added by Heartbeat, then the custom fields overwrite the other diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index c5565e7ee9d..3c671e85296 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -369,7 +369,7 @@ will be overwritten by the value declared here. [float] ==== `fields_under_root` -If this option is set to true, the custom <> +If this option is set to true, the custom <> are stored as top-level fields in the output document instead of being grouped under a `fields` sub-dictionary. If the custom field names conflict with other field names added by Packetbeat, then the custom fields overwrite the other @@ -514,7 +514,7 @@ packetbeat.protocols: [[packetbeat-fields-under-root]] ==== `fields_under_root` -If this option is set to true, the custom <> +If this option is set to true, the custom <> are stored as top-level fields in the output document instead of being grouped under a `fields` sub-dictionary. If the custom field names conflict with other field names added by Packetbeat, then the custom fields overwrite the other diff --git a/winlogbeat/docs/winlogbeat-options.asciidoc b/winlogbeat/docs/winlogbeat-options.asciidoc index 406d5adca54..9c80a4ce029 100644 --- a/winlogbeat/docs/winlogbeat-options.asciidoc +++ b/winlogbeat/docs/winlogbeat-options.asciidoc @@ -335,7 +335,7 @@ winlogbeat.event_logs: [float] ==== `event_logs.fields_under_root` -If this option is set to true, the custom <> +If this option is set to true, the custom <> are stored as top-level fields in the output document instead of being grouped under a `fields` sub-dictionary. If the custom field names conflict with other field names added by Winlogbeat, then the custom fields overwrite the other From 68a6f35a3f9237773b7f29077c3a18e9ca960dbc Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 11 Sep 2017 11:16:32 +0200 Subject: [PATCH 11/96] Update ES test version to 7.0.0-alpha1 (#5142) 6.0.0-beta1 snapshot does not exist as its not a snapshot anymore. Also master should be tested againts ES 7.0.0-alpha1. --- libbeat/dashboards/es_loader_test.go | 6 ++++-- testing/environments/args.yml | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libbeat/dashboards/es_loader_test.go b/libbeat/dashboards/es_loader_test.go index ce039bbf77e..0351acca6ae 100644 --- a/libbeat/dashboards/es_loader_test.go +++ b/libbeat/dashboards/es_loader_test.go @@ -24,7 +24,8 @@ func TestImporter(t *testing.T) { } client := elasticsearch.GetTestingElasticsearch(t) - if strings.HasPrefix(client.Connection.GetVersion(), "6.") { + if strings.HasPrefix(client.Connection.GetVersion(), "6.") || + strings.HasPrefix(client.Connection.GetVersion(), "7.") { t.Skip("Skipping tests for Elasticsearch 6.x releases") } @@ -60,7 +61,8 @@ func TestImporterEmptyBeat(t *testing.T) { } client := elasticsearch.GetTestingElasticsearch(t) - if strings.HasPrefix(client.Connection.GetVersion(), "6.") { + if strings.HasPrefix(client.Connection.GetVersion(), "6.") || + strings.HasPrefix(client.Connection.GetVersion(), "7.") { t.Skip("Skipping tests for Elasticsearch 6.x releases") } diff --git a/testing/environments/args.yml b/testing/environments/args.yml index fd7104210a9..d3064fa6dac 100644 --- a/testing/environments/args.yml +++ b/testing/environments/args.yml @@ -6,5 +6,5 @@ services: build: args: DOWNLOAD_URL: https://snapshots.elastic.co/downloads - ELASTIC_VERSION: 6.0.0-beta1-SNAPSHOT - CACHE_BUST: 20170720 + ELASTIC_VERSION: 7.0.0-alpha1-SNAPSHOT + CACHE_BUST: 20170911 From 0d89cc70d41a4beae76c84dd3766325f0d891e74 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Mon, 11 Sep 2017 20:39:20 +0200 Subject: [PATCH 12/96] Add flush timeout setting to filebeat registrar (#5146) --- CHANGELOG.asciidoc | 1 + filebeat/beater/filebeat.go | 2 +- filebeat/config/config.go | 1 + filebeat/registrar/registrar.go | 47 +++++++++++++++++++++++++-------- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f2efb4799f7..4135deeaa3c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -81,6 +81,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085] - Changed the number of shards in the default configuration to 3. {issue}5095[5095] - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085] +- Add `filebeat.registry_flush` setting, to delay the registry updates. {pull}5146[5146] *Heartbeat* diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index bf4559d2e67..dc67b059a34 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -167,7 +167,7 @@ func (fb *Filebeat) Run(b *beat.Beat) error { finishedLogger := newFinishedLogger(wgEvents) // Setup registrar to persist state - registrar, err := registrar.New(config.RegistryFile, finishedLogger) + registrar, err := registrar.New(config.RegistryFile, config.RegistryFlush, finishedLogger) if err != nil { logp.Err("Could not init registrar: %v", err) return err diff --git a/filebeat/config/config.go b/filebeat/config/config.go index 9e79203be5d..b8dcc1f5ba9 100644 --- a/filebeat/config/config.go +++ b/filebeat/config/config.go @@ -21,6 +21,7 @@ const ( type Config struct { Prospectors []*common.Config `config:"prospectors"` RegistryFile string `config:"registry_file"` + RegistryFlush time.Duration `config:"registry_flush"` ConfigDir string `config:"config_dir"` ShutdownTimeout time.Duration `config:"shutdown_timeout"` Modules []*common.Config `config:"modules"` diff --git a/filebeat/registrar/registrar.go b/filebeat/registrar/registrar.go index 4b535e221ea..5cbd05de5fe 100644 --- a/filebeat/registrar/registrar.go +++ b/filebeat/registrar/registrar.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/elastic/beats/filebeat/input/file" helper "github.com/elastic/beats/libbeat/common/file" @@ -18,9 +19,12 @@ type Registrar struct { Channel chan []file.State out successLogger done chan struct{} - registryFile string // Path to the Registry File - states *file.States // Map with all file paths inside and the corresponding state + registryFile string // Path to the Registry File wg sync.WaitGroup + + states *file.States // Map with all file paths inside and the corresponding state + flushTimeout time.Duration + bufferedStateUpdates int } type successLogger interface { @@ -34,12 +38,13 @@ var ( registryWrites = monitoring.NewInt(nil, "registrar.writes") ) -func New(registryFile string, out successLogger) (*Registrar, error) { +func New(registryFile string, flushTimeout time.Duration, out successLogger) (*Registrar, error) { r := &Registrar{ registryFile: registryFile, done: make(chan struct{}), states: file.NewStates(), Channel: make(chan []file.State, 1), + flushTimeout: flushTimeout, out: out, wg: sync.WaitGroup{}, } @@ -149,13 +154,28 @@ func (r *Registrar) Run() { r.wg.Done() }() + var ( + timer *time.Timer + flushC <-chan time.Time + ) + for { select { case <-r.done: logp.Info("Ending Registrar") return + case <-flushC: + flushC = nil + timer.Stop() + r.flushRegistry() case states := <-r.Channel: r.onEvents(states) + if r.flushTimeout <= 0 { + r.flushRegistry() + } else if flushC == nil { + timer = time.NewTimer(r.flushTimeout) + flushC = timer.C + } } } } @@ -168,17 +188,11 @@ func (r *Registrar) onEvents(states []file.State) { cleanedStates := r.states.Cleanup() statesCleanup.Add(int64(cleanedStates)) + r.bufferedStateUpdates += len(states) + logp.Debug("registrar", "Registrar states cleaned up. Before: %d, After: %d", beforeCount, beforeCount-cleanedStates) - - if err := r.writeRegistry(); err != nil { - logp.Err("Writing of registry returned error: %v. Continuing...", err) - } - - if r.out != nil { - r.out.Published(len(states)) - } } // processEventStates gets the states from the events and writes them to the registrar state @@ -198,6 +212,17 @@ func (r *Registrar) Stop() { r.wg.Wait() } +func (r *Registrar) flushRegistry() { + if err := r.writeRegistry(); err != nil { + logp.Err("Writing of registry returned error: %v. Continuing...", err) + } + + if r.out != nil { + r.out.Published(r.bufferedStateUpdates) + } + r.bufferedStateUpdates = 0 +} + // writeRegistry writes the new json registry file to disk. func (r *Registrar) writeRegistry() error { logp.Debug("registrar", "Write registry file: %s", r.registryFile) From ccc4ab13bc675401411402cb11156e599a08c7f8 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Mon, 11 Sep 2017 13:17:13 -0700 Subject: [PATCH 13/96] [Docs] Fix incorrect ES output config example (#5118) --- libbeat/docs/outputconfig.asciidoc | 3 --- 1 file changed, 3 deletions(-) diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index a2443fc8abe..dc71dc25cab 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -41,9 +41,6 @@ Example configuration: output.elasticsearch: hosts: ["http://localhost:9200"] - template.enabled: true - template.fields: "fields.yml" - template.overwrite: false index: "{beatname_lc}" ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] ssl.certificate: "/etc/pki/client/cert.pem" From c2d7b13d45d4c8ee033fabdb8344089a6264ede2 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Mon, 11 Sep 2017 13:18:08 -0700 Subject: [PATCH 14/96] [Docs] Add option to log messages in JSON (#4931) * [Docs] Add option to log messages in JSON * Add logging.files.permissions to full reference yml file Edit description of file permissions --- auditbeat/auditbeat.reference.yml | 4 ++++ filebeat/filebeat.reference.yml | 4 ++++ heartbeat/heartbeat.reference.yml | 4 ++++ libbeat/_meta/config.reference.yml | 4 ++++ libbeat/docs/loggingconfig.asciidoc | 32 +++++++++++++++++------------ metricbeat/metricbeat.reference.yml | 4 ++++ packetbeat/packetbeat.reference.yml | 4 ++++ winlogbeat/winlogbeat.reference.yml | 4 ++++ 8 files changed, 47 insertions(+), 13 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 2e3a127379c..7ad40e0c424 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -813,5 +813,9 @@ logging.files: # Number of rotated log files to keep. Oldest files will be deleted first. #keepfiles: 7 + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + # Set to true to log messages in json format. #logging.json: false diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 1bd319ca0fc..11f0fe833be 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1233,5 +1233,9 @@ logging.files: # Number of rotated log files to keep. Oldest files will be deleted first. #keepfiles: 7 + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + # Set to true to log messages in json format. #logging.json: false diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e17e962120e..ac73a44c4f1 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -962,5 +962,9 @@ logging.files: # Number of rotated log files to keep. Oldest files will be deleted first. #keepfiles: 7 + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + # Set to true to log messages in json format. #logging.json: false diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 28bf014bbb4..8f69c25573d 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -748,5 +748,9 @@ logging.files: # Number of rotated log files to keep. Oldest files will be deleted first. #keepfiles: 7 + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + # Set to true to log messages in json format. #logging.json: false diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 3383f0c2d59..5ed329723a9 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -40,12 +40,12 @@ the logging output configuration from the command line. See You can specify the following options in the `logging` section of the +{beatname_lc}.yml+ config file: [float] -==== `to_syslog` +==== `logging.to_syslog` When true, writes all logging output to the syslog. [float] -==== `to_files` +==== `logging.to_files` When true, writes all logging output to files. The log files are automatically rotated when the log file size limit is reached. @@ -56,7 +56,7 @@ there will be no log file in the directory specified for logs. [float] [[level]] -==== `level` +==== `logging.level` Minimum log level. One of `debug`, `info`, `warning`, `error`, or `critical`. The default log level is `info`. @@ -79,7 +79,7 @@ that are published. Also logs any warnings, errors, or critical errors. [float] [[selectors]] -==== `selectors` +==== `logging.selectors` The list of debugging-only selector tags used by different Beats components. Use `*` to enable debug output for all components. For example add `publish` to display @@ -88,7 +88,7 @@ selectors can be overwritten using the `-d` command line option (`-d` also sets the debug log level). [float] -==== `metrics.enabled` +==== `logging.metrics.enabled` If enabled, {beatname_uc} periodically logs its internal metrics that have changed in the last period. For each metric that changed, the delta from the @@ -107,48 +107,54 @@ metrics and for this reason they are also not documented. [float] -==== `metrics.period` +==== `logging.metrics.period` The period after which to log the internal metrics. The default is 30s. [float] -==== `files.path` +==== `logging.files.path` The directory that log files are written to. The default is the logs path. See the <> section for details. [float] -==== `files.name` +==== `logging.files.name` The name of the file that logs are written to. By default, the name of the Beat is used. [float] -==== `files.rotateeverybytes` +==== `logging.files.rotateeverybytes` The maximum size of a log file. If the limit is reached, a new log file is generated. The default size limit is 10485760 (10 MB). [float] -==== `files.keepfiles` +==== `logging.files.keepfiles` The number of most recent rotated log files to keep on disk. Older files are deleted during log rotation. The default value is 7. The `keepfiles` options has to be in the range of 2 to 1024 files. [float] -==== `files.permissions` +==== `logging.files.permissions` -The permission mask to apply when rotating log files. The default value is 0600. The - `permissions` options must be a valid Unix-style file permissions mask expressed +The permissions mask to apply when rotating log files. The default value is 0600. The + `permissions` option must be a valid Unix-style file permissions mask expressed in octal notation. In Go, numbers in octal notation must start with '0'. Examples: + * 0644: give read and write access to the file owner, and read access to all others. * 0600: give read and write access to the file owner, and no access to all others. * 0664: give read and write access to the file owner and members of the group associated with the file, as well as read access to all other users. +[float] +==== `logging.json` + +When true, logs messages in JSON format. The default is false. + [float] === Logging format diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 12766fa9a2b..0a9a09b6fa1 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1192,5 +1192,9 @@ logging.files: # Number of rotated log files to keep. Oldest files will be deleted first. #keepfiles: 7 + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + # Set to true to log messages in json format. #logging.json: false diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 0beec09d542..633549190e5 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1200,5 +1200,9 @@ logging.files: # Number of rotated log files to keep. Oldest files will be deleted first. #keepfiles: 7 + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + # Set to true to log messages in json format. #logging.json: false diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 4454e8b44d7..6d2a0570081 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -777,5 +777,9 @@ logging.files: # Number of rotated log files to keep. Oldest files will be deleted first. #keepfiles: 7 + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + # Set to true to log messages in json format. #logging.json: false From b247e88f59de3e7b0709b7441e80978855fb8dbc Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Tue, 12 Sep 2017 01:02:43 -0400 Subject: [PATCH 15/96] Remove runner creation from every reload check (#5141) --- CHANGELOG.asciidoc | 1 + libbeat/cfgfile/reload.go | 44 ++++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 4135deeaa3c..b4c94ff9474 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -37,6 +37,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Combine `fields.yml` properties when they are defined in different sources. {issue}5075[5075] - Keep Docker & Kubernetes pod metadata after container dies while they are needed by processors. {pull}5084[5084] - Fix `fields.yml` lookup when using `export template` with a custom `path.config` param. {issue}5089[5089] +- Remove runner creation from every reload check {pull}5141[5141] *Auditbeat* diff --git a/libbeat/cfgfile/reload.go b/libbeat/cfgfile/reload.go index 30319f43851..916b5e70f41 100644 --- a/libbeat/cfgfile/reload.go +++ b/libbeat/cfgfile/reload.go @@ -6,6 +6,7 @@ import ( "time" "github.com/joeshaw/multierror" + "github.com/mitchellh/hashstructure" "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" @@ -134,7 +135,7 @@ func (rl *Reloader) Run(runnerFactory RunnerFactory) { rl.config.Reload.Period = 0 } - overwriteUpate := true + overwriteUpdate := true for { select { @@ -154,8 +155,8 @@ func (rl *Reloader) Run(runnerFactory RunnerFactory) { } // no file changes - if !updated && !overwriteUpate { - overwriteUpate = false + if !updated && !overwriteUpdate { + overwriteUpdate = false continue } @@ -174,29 +175,34 @@ func (rl *Reloader) Run(runnerFactory RunnerFactory) { continue } - runner, err := runnerFactory.Create(c) + rawCfg := map[string]interface{}{} + err := c.Unpack(rawCfg) + + if err != nil { + logp.Err("Unable to unpack config file due to error: %v", err) + continue + } + + hash, err := hashstructure.Hash(rawCfg, nil) if err != nil { // Make sure the next run also updates because some runners were not properly loaded - overwriteUpate = true - - // In case prospector already is running, do not stop it - if runner != nil && rl.registry.Has(runner.ID()) { - debugf("Remove module from stoplist: %v", runner.ID()) - delete(stopList, runner.ID()) - } else { - logp.Err("Error creating module: %s", err) - } + overwriteUpdate = true + debugf("Unable to generate hash for config file %v due to error: %v", c, err) continue } - debugf("Remove module from stoplist: %v", runner.ID()) - delete(stopList, runner.ID()) + debugf("Remove module from stoplist: %v", hash) + delete(stopList, hash) // As module already exist, it must be removed from the stop list and not started - if !rl.registry.Has(runner.ID()) { - debugf("Add module to startlist: %v", runner.ID()) - startList[runner.ID()] = runner - continue + if !rl.registry.Has(hash) { + debugf("Add module to startlist: %v", hash) + runner, err := runnerFactory.Create(c) + if err != nil { + logp.Err("Unable to create runner due to error: %v", err) + continue + } + startList[hash] = runner } } From 60826036b13358b3189c53f4f2d99eda01489cc7 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Tue, 12 Sep 2017 14:45:12 +0200 Subject: [PATCH 16/96] Reorder processors in publisher pipeline (#5149) * Reorder processors pipeline order - reorder processors, such that all client processors run before the global processors - remove support for EventsMetadataKey * Combine fields and tags if possible - combine client internal field updates and configured field updates into one dictionary - combine proessors for adding client and global fields+tags if no client processors are configured * Do not copy if no processors are defined Do not Clone (deep copy) fields being added in the pipeline if no processors are configured. As processors might add/remove fields and potentially modify shared field objects, these must be copied if there is a chance global shared structured being overwritten by processors. Especially if processors are guarded by conditions. * Fix processors order once again Problem is local fields+tags must be applied after the globaly configured fields and tests, while client processors must be run before the global ones. * review - use 'disabled' flag - simplify pipeline fields init --- libbeat/common/mapstr.go | 5 +- libbeat/publisher/pipeline/pipeline.go | 76 ++++++---- libbeat/publisher/pipeline/processor.go | 183 +++++++++++------------- metricbeat/mb/module/event.go | 1 - metricbeat/mb/testing/data_generator.go | 3 - 5 files changed, 134 insertions(+), 134 deletions(-) diff --git a/libbeat/common/mapstr.go b/libbeat/common/mapstr.go index c38977d10f2..3bf851266e7 100644 --- a/libbeat/common/mapstr.go +++ b/libbeat/common/mapstr.go @@ -11,9 +11,8 @@ import ( // Event metadata constants. These keys are used within libbeat to identify // metadata stored in an event. const ( - EventMetadataKey = "_event_metadata" - FieldsKey = "fields" - TagsKey = "tags" + FieldsKey = "fields" + TagsKey = "tags" ) var ( diff --git a/libbeat/publisher/pipeline/pipeline.go b/libbeat/publisher/pipeline/pipeline.go index e3ac209580c..9c88d22fe34 100644 --- a/libbeat/publisher/pipeline/pipeline.go +++ b/libbeat/publisher/pipeline/pipeline.go @@ -60,10 +60,13 @@ type pipelineProcessors struct { // The pipeline its processor settings for // constructing the clients complete processor // pipeline on connect. - beatMetaProcessor beat.Processor - eventMetaProcessor beat.Processor - processors beat.Processor - disabled bool // disabled is set if outputs have been disabled via CLI + beatsMeta common.MapStr + fields common.MapStr + tags []string + + processors beat.Processor + + disabled bool // disabled is set if outputs have been disabled via CLI } // Settings is used to pass additional settings to a newly created pipeline instance. @@ -132,40 +135,18 @@ func New( out outputs.Group, settings Settings, ) (*Pipeline, error) { - annotations := settings.Annotations var err error - var beatMeta beat.Processor - if meta := annotations.Beat; meta != nil { - beatMeta = beatAnnotateProcessor(meta) - } - - var eventMeta beat.Processor - if em := annotations.Event; len(em.Fields) > 0 || len(em.Tags) > 0 { - eventMeta = eventAnnotateProcessor(em) - } - - var prog beat.Processor - if ps := settings.Processors; ps != nil && len(ps.List) > 0 { - tmp := &program{title: "global"} - for _, p := range ps.List { - tmp.add(p) - } - prog = tmp - } - log := defaultLogger + annotations := settings.Annotations + processors := settings.Processors + disabledOutput := settings.Disabled p := &Pipeline{ logger: log, observer: nilObserver, waitCloseMode: settings.WaitCloseMode, waitCloseTimeout: settings.WaitClose, - processors: pipelineProcessors{ - beatMetaProcessor: beatMeta, - eventMetaProcessor: eventMeta, - processors: prog, - disabled: settings.Disabled, - }, + processors: makePipelineProcessors(annotations, processors, disabledOutput), } p.ackBuilder = &pipelineEmptyACK{p} p.ackActive = atomic.MakeBool(true) @@ -386,3 +367,38 @@ func (e *waitCloser) dec(n int) { func (e *waitCloser) wait() { e.events.Wait() } + +func makePipelineProcessors( + annotations Annotations, + processors *processors.Processors, + disabled bool, +) pipelineProcessors { + p := pipelineProcessors{ + disabled: disabled, + } + + hasProcessors := processors != nil && len(processors.List) > 0 + if hasProcessors { + tmp := &program{title: "global"} + for _, p := range processors.List { + tmp.add(p) + } + p.processors = tmp + } + + if meta := annotations.Beat; meta != nil { + p.beatsMeta = common.MapStr{"beat": meta} + } + + if em := annotations.Event; len(em.Fields) > 0 { + fields := common.MapStr{} + common.MergeFields(fields, em.Fields.Clone(), em.FieldsUnderRoot) + p.fields = fields + } + + if t := annotations.Event.Tags; len(t) > 0 { + p.tags = t + } + + return p +} diff --git a/libbeat/publisher/pipeline/processor.go b/libbeat/publisher/pipeline/processor.go index 597af454011..209150f5821 100644 --- a/libbeat/publisher/pipeline/processor.go +++ b/libbeat/publisher/pipeline/processor.go @@ -29,73 +29,76 @@ type processorFn struct { // // Pipeline (C=client, P=pipeline) // -// 1. (P) extract EventMetadataKey fields + tags (to be removed in favor of 4) -// 2. (P) generalize/normalize event -// 3. (P) add beats metadata (name, hostname, version) -// 4. (C) add Meta from client Config to event.Meta -// 5. (C) add Fields from client config to event.Fields -// 6. (P) add pipeline fields + tags -// 7. (C) add client fields + tags -// 8. (P/C) apply EventMetadataKey fields + tags (to be removed in favor of 4) -// 9. (C) client processors list -// 10. (P) pipeline processors list -// 11. (P) (if publish/debug enabled) log event -// 12. (P) (if output disabled) dropEvent +// 1. (P) generalize/normalize event +// 2. (C) add Meta from client Config to event.Meta +// 3. (C) add Fields from client config to event.Fields +// 4. (P) add pipeline fields + tags +// 5. (C) add client fields + tags +// 6. (C) client processors list +// 7. (P) add beats metadata +// 8. (P) pipeline processors list +// 9. (P) (if publish/debug enabled) log event +// 10. (P) (if output disabled) dropEvent func (p *Pipeline) newProcessorPipeline( config beat.ClientConfig, ) beat.Processor { - processors := &program{title: "processPipeline"} + var ( + // pipeline processors + processors = &program{title: "processPipeline"} - global := p.processors + // client fields and metadata + clientMeta = config.Meta + localProcessors = makeClientProcessors(config) - // setup 1: extract EventMetadataKey fields + tags - processors.add(preEventUserAnnotateProcessor) + // pipeline global + global = p.processors + ) - // setup 2 and 3: generalize/normalize output (P) + needsCopy := localProcessors != nil || global.processors != nil + + // setup 1: generalize/normalize output (P) processors.add(generalizeProcessor) - processors.add(global.beatMetaProcessor) - // setup 4: add Meta from client config - if m := config.Meta; len(m) > 0 { - processors.add(clientEventMeta(m)) + // setup 2: add Meta from client config (C) + if m := clientMeta; len(m) > 0 { + processors.add(clientEventMeta(m, needsCopy)) } - // setup 5: add Fields from client config - if m := config.Fields; len(m) > 0 { - processors.add(clientEventFields(m)) + // setup 4, 5: pipeline tags + client tags + var tags []string + tags = append(tags, global.tags...) + tags = append(tags, config.EventMetadata.Tags...) + if len(tags) > 0 { + processors.add(makeAddTagsProcessor("tags", tags)) } - // setup 6: add event fields + tags (P) - processors.add(global.eventMetaProcessor) - - // setup 7: add fields + tags (C) - if em := config.EventMetadata; len(em.Fields) > 0 || len(em.Tags) > 0 { - processors.add(eventAnnotateProcessor(em)) + // setup 3, 4, 5: client config fields + pipeline fields + client fields + fields := config.Fields.Clone() + fields.DeepUpdate(global.fields) + if em := config.EventMetadata; len(em.Fields) > 0 { + common.MergeFields(fields, em.Fields.Clone(), em.FieldsUnderRoot) + } + if len(fields) > 0 { + processors.add(makeAddFieldsProcessor("fields", fields, needsCopy)) } - // setup 8: apply EventMetadata fields + tags - processors.add(eventUserAnnotateProcessor) - - // setup 9: client processors (C) - if procs := config.Processor; procs != nil { - if lst := procs.All(); len(lst) > 0 { + // setup 5: client processor list + processors.add(localProcessors) - processors.add(&program{ - title: "client", - list: lst, - }) - } + // setup 6: add beats metadata + if meta := global.beatsMeta; len(meta) > 0 { + processors.add(makeAddFieldsProcessor("beatsMeta", meta, needsCopy)) } - // setup 10: pipeline processors (P) + // setup 7: pipeline processors list processors.add(global.processors) - // setup 11: debug print final event (P) + // setup 9: debug print final event (P) if logp.IsDebug("publish") { processors.add(debugPrintProcessor()) } - // setup 12: drop all events if outputs are disabled + // setup 10: drop all events if outputs are disabled (P) if global.disabled { processors.add(dropDisabledProcessor) } @@ -205,67 +208,41 @@ func eventAnnotateProcessor(eventMeta common.EventMetadata) *processorFn { }) } -func clientEventMeta(meta common.MapStr) *processorFn { - return newAnnotateProcessor("@metadata", func(event *beat.Event) { - if event.Meta == nil { - event.Meta = meta.Clone() - } else { - event.Meta = event.Meta.Clone() - event.Meta.DeepUpdate(meta.Clone()) - } - }) +func clientEventMeta(meta common.MapStr, needsCopy bool) *processorFn { + fn := func(event *beat.Event) { addMeta(event, meta) } + if needsCopy { + fn = func(event *beat.Event) { addMeta(event, meta.Clone()) } + } + return newAnnotateProcessor("@metadata", fn) } -func clientEventFields(fields common.MapStr) *processorFn { - return newAnnotateProcessor("globalFields", func(event *beat.Event) { - event.Fields.DeepUpdate(fields.Clone()) - }) +func addMeta(event *beat.Event, meta common.MapStr) { + if event.Meta == nil { + event.Meta = meta + } else { + event.Meta.Clone() + event.Meta.DeepUpdate(meta) + } } -// TODO: remove var-section. Keep for backwards compatibility with old publisher API. -// Remove after updating all beats to new publisher API. -// Note: this functionality is used by filebeat/winlogbeat, so prospector/harvesters -// can apply fields to events after generating the event type. -// This functionality will be removed, in favor of harvesters publishing -// event to a beat.Client with properly setup processor -var ( - preEventUserAnnotateProcessor = newAnnotateProcessor("annotateEventUserPre", func(event *beat.Event) { - const key = common.EventMetadataKey - val, exists := event.Fields[key] - if !exists { - return - } - - delete(event.Fields, key) +func pipelineEventFields(fields common.MapStr, copy bool) *processorFn { + return makeAddFieldsProcessor("pipelineFields", fields, copy) +} - if _, ok := val.(common.EventMetadata); ok { - if event.Meta == nil { - event.Meta = common.MapStr{} - } - event.Meta[key] = val - } +func makeAddTagsProcessor(name string, tags []string) *processorFn { + return newAnnotateProcessor(name, func(event *beat.Event) { + common.AddTags(event.Fields, tags) }) +} - eventUserAnnotateProcessor = newAnnotateProcessor("annotateEventUser", func(event *beat.Event) { - const key = common.EventMetadataKey - - tmp, ok := event.Meta[key] - if !ok { - return - } - - delete(event.Meta, key) - if len(event.Meta) == 0 { - event.Meta = nil - } +func makeAddFieldsProcessor(name string, fields common.MapStr, copy bool) *processorFn { + fn := func(event *beat.Event) { event.Fields.DeepUpdate(fields) } + if copy { + fn = func(event *beat.Event) { event.Fields.DeepUpdate(fields.Clone()) } + } - eventMeta := tmp.(common.EventMetadata) - common.AddTags(event.Fields, eventMeta.Tags) - if fields := eventMeta.Fields; len(fields) > 0 { - common.MergeFields(event.Fields, fields.Clone(), eventMeta.FieldsUnderRoot) - } - }) -) + return newAnnotateProcessor(name, fn) +} func debugPrintProcessor() *processorFn { // ensure only one go-routine is using the encoder (in case @@ -286,3 +263,15 @@ func debugPrintProcessor() *processorFn { return event, nil }) } + +func makeClientProcessors(config beat.ClientConfig) processors.Processor { + procs := config.Processor + if procs == nil || len(procs.All()) == 0 { + return nil + } + + return &program{ + title: "client", + list: procs.All(), + } +} diff --git a/metricbeat/mb/module/event.go b/metricbeat/mb/module/event.go index 3b3c1bd0e93..c9434cdff15 100644 --- a/metricbeat/mb/module/event.go +++ b/metricbeat/mb/module/event.go @@ -78,7 +78,6 @@ func (b EventBuilder) Build() (beat.Event, error) { event := beat.Event{ Timestamp: time.Time(timestamp), Fields: common.MapStr{ - // common.EventMetadataKey: b.metadata, b.ModuleName: moduleEvent, "metricset": metricsetData, }, diff --git a/metricbeat/mb/testing/data_generator.go b/metricbeat/mb/testing/data_generator.go index 1fe83e50977..77ad9922c84 100644 --- a/metricbeat/mb/testing/data_generator.go +++ b/metricbeat/mb/testing/data_generator.go @@ -85,9 +85,6 @@ func CreateFullEvent(ms mb.MetricSet, metricSetData common.MapStr) beat.Event { "hostname": "host.example.com", } - // Delete meta data as not needed for the event output here. - delete(fullEvent.Fields, common.EventMetadataKey) - return fullEvent } From ffd23a9145e093d5e5e0b963a01d69a95b51589a Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Tue, 12 Sep 2017 14:56:35 +0200 Subject: [PATCH 17/96] Add specialized buffers to memqueue (#5148) * memqueue cleanup Move the buffer and message handling from memqueue broker type to the actual go routines handling the messaging. * Add specialized buffers to memqueue Use specialized buffer types and eventloops depending on the memqueue configuration. If flushing is disabled, a region based ring buffer is used, as loads of small batches can be generated and we want to minimize additional allocations of small objects. If flushing is enabled a list of active and flushed buffers is managed by the queue. If a buffer is flushed by timeout, but not yet processed (and not full), additional events being published are added to the already flushed buffer. * ensure flush list does not contain empty buffers --- libbeat/publisher/queue/memqueue/ackloop.go | 61 ++- libbeat/publisher/queue/memqueue/batchbuf.go | 65 +++ libbeat/publisher/queue/memqueue/broker.go | 165 ++---- libbeat/publisher/queue/memqueue/buf.go | 278 ---------- libbeat/publisher/queue/memqueue/consume.go | 11 +- libbeat/publisher/queue/memqueue/eventloop.go | 505 ++++++++++++++++-- .../publisher/queue/memqueue/internal_api.go | 4 +- .../publisher/queue/memqueue/queue_test.go | 32 +- libbeat/publisher/queue/memqueue/ringbuf.go | 293 ++++++++++ 9 files changed, 909 insertions(+), 505 deletions(-) create mode 100644 libbeat/publisher/queue/memqueue/batchbuf.go create mode 100644 libbeat/publisher/queue/memqueue/ringbuf.go diff --git a/libbeat/publisher/queue/memqueue/ackloop.go b/libbeat/publisher/queue/memqueue/ackloop.go index f5a117383cc..58722514470 100644 --- a/libbeat/publisher/queue/memqueue/ackloop.go +++ b/libbeat/publisher/queue/memqueue/ackloop.go @@ -7,7 +7,7 @@ package memqueue // Producer ACKs are run in the ackLoop go-routine. type ackLoop struct { broker *Broker - sig chan batchAckRequest + sig chan batchAckMsg lst chanList totalACK uint64 @@ -15,6 +15,14 @@ type ackLoop struct { batchesSched uint64 batchesACKed uint64 + + processACK func(chanList, int) +} + +func newACKLoop(b *Broker, processACK func(chanList, int)) *ackLoop { + l := &ackLoop{broker: b} + l.processACK = processACK + return l } func (l *ackLoop) run() { @@ -72,38 +80,51 @@ func (l *ackLoop) run() { // handleBatchSig collects and handles a batch ACK/Cancel signal. handleBatchSig // is run by the ackLoop. func (l *ackLoop) handleBatchSig() int { + lst := l.collectAcked() + + count := 0 + for current := lst.front(); current != nil; current = current.next { + count += current.count + } + + if e := l.broker.eventer; e != nil { + e.OnACK(count) + } + + // report acks to waiting clients + l.processACK(lst, count) + + for !lst.empty() { + releaseACKChan(lst.pop()) + } + + // return final ACK to EventLoop, in order to clean up internal buffer + l.broker.logger.Debug("ackloop: return ack to broker loop:", count) + + l.totalACK += uint64(count) + l.broker.logger.Debug("ackloop: done send ack") + return count +} + +func (l *ackLoop) collectAcked() chanList { + lst := chanList{} + acks := l.lst.pop() - l.broker.logger.Debugf("ackloop: receive ack [%v: %v, %v]", acks.seq, acks.start, acks.count) - start := acks.start - count := acks.count - l.batchesACKed++ - releaseACKChan(acks) + lst.append(acks) done := false - // collect pending ACKs for !l.lst.empty() && !done { acks := l.lst.front() select { case <-acks.ch: l.broker.logger.Debugf("ackloop: receive ack [%v: %v, %v]", acks.seq, acks.start, acks.count) - - count += acks.count l.batchesACKed++ - releaseACKChan(l.lst.pop()) + lst.append(l.lst.pop()) default: done = true } } - // report acks to waiting clients - states := l.broker.buf.buf.clients - l.broker.reportACK(states, start, count) - - // return final ACK to EventLoop, in order to clean up internal buffer - l.broker.logger.Debug("ackloop: return ack to broker loop:", count) - - l.totalACK += uint64(count) - l.broker.logger.Debug("ackloop: done send ack") - return count + return lst } diff --git a/libbeat/publisher/queue/memqueue/batchbuf.go b/libbeat/publisher/queue/memqueue/batchbuf.go new file mode 100644 index 00000000000..41ff0e20815 --- /dev/null +++ b/libbeat/publisher/queue/memqueue/batchbuf.go @@ -0,0 +1,65 @@ +package memqueue + +import "github.com/elastic/beats/libbeat/publisher" + +type batchBuffer struct { + next *batchBuffer + flushed bool + events []publisher.Event + clients []clientState +} + +func newBatchBuffer(sz int) *batchBuffer { + b := &batchBuffer{} + b.init(sz) + return b +} + +func (b *batchBuffer) init(sz int) { + b.events = make([]publisher.Event, 0, sz) + b.clients = make([]clientState, 0, sz) +} + +func (b *batchBuffer) initWith(sz int, old batchBuffer) { + events, clients := old.events, old.clients + L := len(events) + + b.events = make([]publisher.Event, L, sz) + b.clients = make([]clientState, L, sz) + + copy(b.events, events) + copy(b.clients, clients) +} + +func (b *batchBuffer) add(event publisher.Event, st clientState) { + b.events = append(b.events, event) + b.clients = append(b.clients, st) +} + +func (b *batchBuffer) length() int { + return len(b.events) +} + +func (b *batchBuffer) capacity() int { + return cap(b.events) +} + +func (b *batchBuffer) cancel(st *produceState) int { + events := b.events[:0] + clients := b.clients[:0] + + removed := 0 + for i := range b.clients { + if b.clients[i].state == st { + removed++ + continue + } + + events = append(events, b.events[i]) + clients = append(clients, b.clients[i]) + } + + b.events = events + b.clients = clients + return removed +} diff --git a/libbeat/publisher/queue/memqueue/broker.go b/libbeat/publisher/queue/memqueue/broker.go index f570921f6a9..a74d294c195 100644 --- a/libbeat/publisher/queue/memqueue/broker.go +++ b/libbeat/publisher/queue/memqueue/broker.go @@ -1,13 +1,10 @@ package memqueue import ( - "fmt" - "math" "sync" "time" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/publisher" "github.com/elastic/beats/libbeat/publisher/queue" ) @@ -16,9 +13,10 @@ type Broker struct { logger logger - buf brokerBuffer - minEvents int - idleTimeout time.Duration + bufSize int + // buf brokerBuffer + // minEvents int + // idleTimeout time.Duration // api channels events chan pushRequest @@ -46,9 +44,10 @@ type Settings struct { type ackChan struct { next *ackChan - ch chan batchAckRequest + ch chan batchAckMsg seq uint start, count int // number of events waiting for ACK + states []clientState } type chanList struct { @@ -119,12 +118,20 @@ func NewBroker( eventer: settings.Eventer, } - b.buf.init(logger, sz) - b.minEvents = minEvents - b.idleTimeout = flushTimeout - eventLoop := newEventLoop(b) - ack := &ackLoop{broker: b} + var eventLoop interface { + run() + processACK(chanList, int) + } + + if minEvents > 1 { + eventLoop = newBufferingEventLoop(b, sz, minEvents, flushTimeout) + } else { + eventLoop = newDirectEventLoop(b, sz) + } + + b.bufSize = sz + ack := newACKLoop(b, eventLoop.processACK) b.wg.Add(2) go func() { @@ -149,7 +156,7 @@ func (b *Broker) Close() error { func (b *Broker) BufferConfig() queue.BufferConfig { return queue.BufferConfig{ - Events: b.buf.Size(), + Events: b.bufSize, } } @@ -161,136 +168,21 @@ func (b *Broker) Consumer() queue.Consumer { return newConsumer(b) } -func (b *Broker) insert(req *pushRequest) (int, bool) { - var avail int - if req.state == nil { - _, avail = b.buf.insert(req.event, clientState{}) - } else { - st := req.state - if st.cancelled { - b.logger.Debugf("cancelled producer - ignore event: %v\t%v\t%p", req.event, req.seq, req.state) - - // do not add waiting events if producer did send cancel signal - - if cb := st.dropCB; cb != nil { - cb(req.event.Content) - } - - return -1, false - } - - _, avail = b.buf.insert(req.event, clientState{ - seq: req.seq, - state: st, - }) - } - - return avail, true -} - -func (b *Broker) get(max int) (startIndex int, events []publisher.Event) { - return b.buf.reserve(max) -} - -func (b *Broker) cancel(st *produceState) int { - return b.buf.cancel(st) -} - -func (b *Broker) full() bool { - return b.buf.Full() -} - -func (b *Broker) avail() int { - return b.buf.Avail() -} - -func (b *Broker) totalAvail() int { - return b.buf.TotalAvail() -} - -func (b *Broker) cleanACKs(count int) { - b.buf.ack(count) -} - -func (b *Broker) reportACK(states []clientState, start, N int) { - { - start := time.Now() - b.logger.Debug("handle ACKs: ", N) - defer func() { - b.logger.Debug("handle ACK took: ", time.Since(start)) - }() - } - - if e := b.eventer; e != nil { - e.OnACK(N) - } - - // TODO: global boolean to check if clients will need an ACK - // no need to report ACKs if no client is interested in ACKs - - idx := start + N - 1 - if idx >= len(states) { - idx -= len(states) - } - - total := 0 - for i := N - 1; i >= 0; i-- { - if idx < 0 { - idx = len(states) - 1 - } - - st := &states[idx] - b.logger.Debugf("try ack index: (idx=%v, i=%v, seq=%v)\n", idx, i, st.seq) - - idx-- - if st.state == nil { - b.logger.Debug("no state set") - continue - } - - count := (st.seq - st.state.lastACK) - if count == 0 || count > math.MaxUint32/2 { - // seq number comparison did underflow. This happens only if st.seq has - // allready been acknowledged - // b.logger.Debug("seq number already acked: ", st.seq) - - st.state = nil - continue - } - - b.logger.Debugf("broker ACK events: count=%v, start-seq=%v, end-seq=%v\n", - count, - st.state.lastACK+1, - st.seq, - ) - - total += int(count) - if total > N { - panic(fmt.Sprintf("Too many events acked (expected=%v, total=%v)", - count, total, - )) - } - - st.state.cb(int(count)) - st.state.lastACK = st.seq - st.state = nil - } -} - var ackChanPool = sync.Pool{ New: func() interface{} { return &ackChan{ - ch: make(chan batchAckRequest, 1), + ch: make(chan batchAckMsg, 1), } }, } -func newACKChan(seq uint, start, count int) *ackChan { +func newACKChan(seq uint, start, count int, states []clientState) *ackChan { ch := ackChanPool.Get().(*ackChan) ch.next = nil ch.seq = seq ch.start = start ch.count = count + ch.states = states return ch } @@ -342,7 +234,7 @@ func (l *chanList) front() *ackChan { return l.head } -func (l *chanList) channel() chan batchAckRequest { +func (l *chanList) channel() chan batchAckMsg { if l.head == nil { return nil } @@ -361,3 +253,12 @@ func (l *chanList) pop() *ackChan { ch.next = nil return ch } + +func (l *chanList) reverse() { + tmp := *l + *l = chanList{} + + for !tmp.empty() { + l.prepend(tmp.pop()) + } +} diff --git a/libbeat/publisher/queue/memqueue/buf.go b/libbeat/publisher/queue/memqueue/buf.go index 1386fd97b10..b180473ad09 100644 --- a/libbeat/publisher/queue/memqueue/buf.go +++ b/libbeat/publisher/queue/memqueue/buf.go @@ -1,279 +1 @@ package memqueue - -import ( - "fmt" - - "github.com/elastic/beats/libbeat/publisher" -) - -// Internal event ring buffer. -// The ring is split into 2 regions. -// Region A contains active events to be send to consumers, while region B can -// only be filled by producers, if there is no more space in region A. Splitting -// the ring buffer into regions enables the broker to send batches of type -// []publisher.Event to the consumer without having to copy and/or grow/shrink the -// buffers. -type brokerBuffer struct { - buf eventBuffer - - regA, regB region - reserved int // amount of events in region A actively processed/reserved -} - -type region struct { - index int - size int -} - -type eventBuffer struct { - logger logger - - events []publisher.Event - clients []clientState -} - -type clientState struct { - seq uint32 // event sequence number - state *produceState // the producer it's state used to compute and signal the ACK count -} - -func (b *brokerBuffer) init(log logger, size int) { - *b = brokerBuffer{} - b.buf.init(size) - b.buf.logger = log -} - -func (b *brokerBuffer) insert(event publisher.Event, client clientState) (bool, int) { - // log := b.buf.logger - // log.Debug("insert:") - // log.Debug(" region A:", b.regA) - // log.Debug(" region B:", b.regB) - // log.Debug(" reserved:", b.reserved) - // defer func() { - // log.Debug(" -> region A:", b.regA) - // log.Debug(" -> region B:", b.regB) - // log.Debug(" -> reserved:", b.reserved) - // }() - - // always insert into region B, if region B exists. - // That is, we have 2 regions and region A is currently processed by consumers - if b.regB.size > 0 { - // log.Debug(" - push into B region") - - idx := b.regB.index + b.regB.size - avail := b.regA.index - idx - if avail == 0 { - return false, 0 - } - - b.buf.Set(idx, event, client) - b.regB.size++ - - return true, avail - 1 - } - - // region B does not exist yet, check if region A is available for use - idx := b.regA.index + b.regA.size - // log.Debug(" - index: ", idx) - // log.Debug(" - buffer size: ", b.buf.Len()) - avail := b.buf.Len() - idx - if avail == 0 { // no more space in region A - // log.Debug(" - region A full") - - if b.regA.index == 0 { - // space to create region B, buffer is full - - // log.Debug(" - no space in region B") - - return false, 0 - } - - // create region B and insert events - // log.Debug(" - create region B") - b.regB.index = 0 - b.regB.size = 1 - b.buf.Set(0, event, client) - return true, b.regA.index - 1 - } - - // space available in region A -> let's append the event - // log.Debug(" - push into region A") - b.buf.Set(idx, event, client) - b.regA.size++ - return true, avail - 1 -} - -// cancel removes all buffered events matching `st`, not yet reserved by -// any consumer -func (b *brokerBuffer) cancel(st *produceState) int { - // log := b.buf.logger - // log.Debug("cancel:") - // log.Debug(" region A:", b.regA) - // log.Debug(" region B:", b.regB) - // log.Debug(" reserved:", b.reserved) - // defer func() { - // log.Debug(" -> region A:", b.regA) - // log.Debug(" -> region B:", b.regB) - // log.Debug(" -> reserved:", b.reserved) - // }() - - // TODO: return if st has no pending events - - cancelB := b.cancelRegion(st, b.regB) - b.regB.size -= cancelB - - cancelA := b.cancelRegion(st, region{ - index: b.regA.index + b.reserved, - size: b.regA.size - b.reserved, - }) - b.regA.size -= cancelA - - return cancelA + cancelB -} - -func (b *brokerBuffer) cancelRegion(st *produceState, reg region) (removed int) { - start := reg.index - end := start + reg.size - events := b.buf.events[start:end] - clients := b.buf.clients[start:end] - - toEvents := events[:0] - toClients := clients[:0] - - // filter loop - for i := 0; i < reg.size; i++ { - if clients[i].state == st { - continue // remove - } - - toEvents = append(toEvents, events[i]) - toClients = append(toClients, clients[i]) - } - - // re-initialize old buffer elements to help garbage collector - events = events[len(toEvents):] - clients = clients[len(toClients):] - for i := range events { - events[i] = publisher.Event{} - clients[i] = clientState{} - } - - return len(events) -} - -// activeBufferOffsets returns start and end offset -// of all available events in region A. -func (b *brokerBuffer) activeBufferOffsets() (int, int) { - return b.regA.index, b.regA.index + b.regA.size -} - -// reserve returns up to `sz` events from the brokerBuffer, -// exclusively marking the events as 'reserved'. Subsequent calls to `reserve` -// will only return enqueued and non-reserved events from the buffer. -// If `sz == -1`, all available events will be reserved. -func (b *brokerBuffer) reserve(sz int) (int, []publisher.Event) { - // log := b.buf.logger - // log.Debug("reserve: ", sz) - // log.Debug(" region A:", b.regA) - // log.Debug(" region B:", b.regB) - // log.Debug(" reserved:", b.reserved) - // defer func() { - // log.Debug(" -> region A:", b.regA) - // log.Debug(" -> region B:", b.regB) - // log.Debug(" -> reserved:", b.reserved) - // }() - - use := b.regA.size - b.reserved - // log.Debug(" - avail: ", use) - - if sz > 0 { - if use > sz { - use = sz - } - } - - start := b.regA.index + b.reserved - end := start + use - b.reserved += use - // log.Debug(" - start:", start) - // log.Debug(" - end:", end) - return start, b.buf.events[start:end] -} - -// ack up to sz events in region A -func (b *brokerBuffer) ack(sz int) { - // log := b.buf.logger - // log.Debug("ack: ", sz) - // log.Debug(" region A:", b.regA) - // log.Debug(" region B:", b.regB) - // log.Debug(" reserved:", b.reserved) - // defer func() { - // log.Debug(" -> region A:", b.regA) - // log.Debug(" -> region B:", b.regB) - // log.Debug(" -> reserved:", b.reserved) - // }() - - if b.regA.size < sz { - panic(fmt.Errorf("Commit region to big (commit region=%v, buffer size=%v)", - sz, b.regA.size, - )) - } - - // clear region, so published events can be collected by the garbage collector: - end := b.regA.index + sz - for i := b.regA.index; i < end; i++ { - b.buf.events[i] = publisher.Event{} - } - - b.regA.index = end - b.regA.size -= sz - b.reserved -= sz - if b.regA.size == 0 { - // region A is empty, transfer region B into region A - b.regA = b.regB - b.regB.index = 0 - b.regB.size = 0 - } -} - -func (b *brokerBuffer) Empty() bool { - return (b.regA.size - b.reserved) == 0 -} - -func (b *brokerBuffer) Avail() int { - return b.regA.size - b.reserved -} - -func (b *brokerBuffer) TotalAvail() int { - return b.regA.size + b.regB.size - b.reserved -} - -func (b *brokerBuffer) Full() bool { - var avail int - if b.regB.size > 0 { - avail = b.regA.index - b.regB.index - b.regB.size - } else { - avail = b.buf.Len() - b.regA.index - b.regA.size - } - return avail == 0 -} - -func (b *brokerBuffer) Size() int { - return b.buf.Len() -} - -func (b *eventBuffer) init(size int) { - b.events = make([]publisher.Event, size) - b.clients = make([]clientState, size) -} - -func (b *eventBuffer) Len() int { - return len(b.events) -} - -func (b *eventBuffer) Set(idx int, event publisher.Event, st clientState) { - // b.logger.Debugf("insert event: idx=%v, seq=%v\n", idx, st.seq) - - b.events[idx] = event - b.clients[idx] = st -} diff --git a/libbeat/publisher/queue/memqueue/consume.go b/libbeat/publisher/queue/memqueue/consume.go index 560c58e4a80..0a9159bca62 100644 --- a/libbeat/publisher/queue/memqueue/consume.go +++ b/libbeat/publisher/queue/memqueue/consume.go @@ -24,10 +24,11 @@ type consumerStats struct { } type batch struct { - consumer *consumer - events []publisher.Event - ack *ackChan - state ackState + consumer *consumer + events []publisher.Event + clientStates []clientState + ack *ackChan + state ackState } type ackState uint8 @@ -112,5 +113,5 @@ func (b *batch) ACK() { } func (b *batch) report() { - b.ack.ch <- batchAckRequest{} + b.ack.ch <- batchAckMsg{} } diff --git a/libbeat/publisher/queue/memqueue/eventloop.go b/libbeat/publisher/queue/memqueue/eventloop.go index 773317e5254..fd3edca15c3 100644 --- a/libbeat/publisher/queue/memqueue/eventloop.go +++ b/libbeat/publisher/queue/memqueue/eventloop.go @@ -1,12 +1,43 @@ package memqueue import ( + "fmt" + "math" "time" ) -type eventLoop struct { +// directEventLoop implements the broker main event loop. It buffers events, +// but tries to forward events as early as possible. +type directEventLoop struct { broker *Broker + buf ringBuffer + + // active broker API channels + events chan pushRequest + get chan getRequest + pubCancel chan producerCancelRequest + + // ack handling + acks chan int // ackloop -> eventloop : total number of events ACKed by outputs + schedACKS chan chanList // eventloop -> ackloop : active list of batches to be acked + pendingACKs chanList // ordered list of active batches to be send to the ackloop + ackSeq uint // ack batch sequence number to validate ordering +} + +// bufferingEventLoop implements the broker main event loop. +// Events in the buffer are forwarded to consumers only if the buffer is full or on flush timeout. +type bufferingEventLoop struct { + broker *Broker + + buf *batchBuffer + flushList flushList + eventCount int + + minEvents int + maxEvents int + flushTimeout time.Duration + // active broker API channels events chan pushRequest get chan getRequest @@ -19,33 +50,35 @@ type eventLoop struct { ackSeq uint // ack batch sequence number to validate ordering // buffer flush timer state - timer *time.Timer - idleC <-chan time.Time - forceFlush bool + timer *time.Timer + idleC <-chan time.Time +} + +type flushList struct { + head *batchBuffer + tail *batchBuffer + count int } -func newEventLoop(b *Broker) *eventLoop { - l := &eventLoop{ +func newDirectEventLoop(b *Broker, size int) *directEventLoop { + l := &directEventLoop{ broker: b, events: b.events, + get: nil, pubCancel: b.pubCancel, acks: b.acks, } - - if to := b.idleTimeout; to > 0 { - // create initialy 'stopped' timer -> reset will be used - // on timer object, if flush timer becomes active. - l.timer = time.NewTimer(to) - if !l.timer.Stop() { - <-l.timer.C - } - } + l.buf.init(b.logger, size) return l } -func (l *eventLoop) run() { - broker := l.broker +func (l *directEventLoop) run() { + var ( + broker = l.broker + buf = &l.buf + ) + for { select { case <-broker.done: @@ -60,10 +93,6 @@ func (l *eventLoop) run() { case req := <-l.get: // consumer asking for next batch l.handleConsumer(&req) - case <-l.idleC: - // handle flush timer being triggered -> pending events can be forwarded via 'get' - l.enableFlushEvents() - case l.schedACKS <- l.pendingACKs: // on send complete list of pending batches has been forwarded -> clear list and queue l.schedACKS = nil @@ -75,25 +104,18 @@ func (l *eventLoop) run() { } // update get and idle timer after state machine - l.get = broker.requests - if !l.forceFlush { - avail := broker.avail() - if avail == 0 || broker.totalAvail() < broker.minEvents { - l.get = nil - - if avail > 0 { - l.startFlushTimer() - } - } + l.get = nil + if buf.Avail() > 0 { + l.get = broker.requests } } } -func (l *eventLoop) handleInsert(req *pushRequest) { +func (l *directEventLoop) handleInsert(req *pushRequest) { // log := l.broker.logger // log.Debugf("push event: %v\t%v\t%p\n", req.event, req.seq, req.state) - if avail, ok := l.broker.insert(req); ok && avail == 0 { + if avail, ok := l.insert(req); ok && avail == 0 { // log.Debugf("buffer: all regions full") // no more space to accept new events -> unset events queue for time being @@ -101,7 +123,30 @@ func (l *eventLoop) handleInsert(req *pushRequest) { } } -func (l *eventLoop) handleCancel(req *producerCancelRequest) { +func (l *directEventLoop) insert(req *pushRequest) (int, bool) { + var avail int + log := l.broker.logger + + if req.state == nil { + _, avail = l.buf.insert(req.event, clientState{}) + return avail, true + } + + st := req.state + if st.cancelled { + reportCancelledState(log, req) + return -1, false + } + + _, avail = l.buf.insert(req.event, clientState{ + seq: req.seq, + state: st, + }) + + return avail, true +} + +func (l *directEventLoop) handleCancel(req *producerCancelRequest) { // log := l.broker.logger // log.Debug("handle cancel request") @@ -112,61 +157,309 @@ func (l *eventLoop) handleCancel(req *producerCancelRequest) { if st := req.state; st != nil { st.cancelled = true - removed = broker.cancel(st) + removed = l.buf.cancel(st) } // signal cancel request being finished if req.resp != nil { - req.resp <- producerCancelResponse{ - removed: removed, - } + req.resp <- producerCancelResponse{removed: removed} } // re-enable pushRequest if buffer can take new events - if !broker.full() { + if !l.buf.Full() { l.events = broker.events } } -func (l *eventLoop) handleConsumer(req *getRequest) { +func (l *directEventLoop) handleConsumer(req *getRequest) { // log := l.broker.logger + // log.Debugf("try reserve %v events", req.sz) - start, buf := l.broker.get(req.sz) + start, buf := l.buf.reserve(req.sz) count := len(buf) if count == 0 { panic("empty batch returned") } // log.Debug("newACKChan: ", b.ackSeq, count) - ackCH := newACKChan(l.ackSeq, start, count) + ackCH := newACKChan(l.ackSeq, start, count, l.buf.buf.clients) l.ackSeq++ - req.resp <- getResponse{buf, ackCH} + req.resp <- getResponse{ackCH, buf} l.pendingACKs.append(ackCH) l.schedACKS = l.broker.scheduledACKs - - l.stopFlushTimer() } -func (l *eventLoop) handleACK(count int) { +func (l *directEventLoop) handleACK(count int) { // log := l.broker.logger // log.Debug("receive buffer ack:", count) // Give broker/buffer a chance to clean up most recent ACKs // After handling ACKs some buffer has been freed up // -> always reenable producers - broker := l.broker - broker.cleanACKs(count) + l.buf.ack(count) l.events = l.broker.events } -func (l *eventLoop) enableFlushEvents() { - l.forceFlush = true - l.idleC = nil +// processACK is used by the ackLoop to process the list of acked batches +func (l *directEventLoop) processACK(lst chanList, N int) { + log := l.broker.logger + { + start := time.Now() + log.Debug("handle ACKs: ", N) + defer func() { + log.Debug("handle ACK took: ", time.Since(start)) + }() + } + + acks := lst.front() + start := acks.start + states := acks.states + + // TODO: global boolean to check if clients will need an ACK + // no need to report ACKs if no client is interested in ACKs + + idx := start + N - 1 + if idx >= len(states) { + idx -= len(states) + } + + total := 0 + for i := N - 1; i >= 0; i-- { + if idx < 0 { + idx = len(states) - 1 + } + + st := &states[idx] + log.Debugf("try ack index: (idx=%v, i=%v, seq=%v)\n", idx, i, st.seq) + + idx-- + if st.state == nil { + log.Debug("no state set") + continue + } + + count := (st.seq - st.state.lastACK) + if count == 0 || count > math.MaxUint32/2 { + // seq number comparison did underflow. This happens only if st.seq has + // allready been acknowledged + // log.Debug("seq number already acked: ", st.seq) + + st.state = nil + continue + } + + log.Debugf("broker ACK events: count=%v, start-seq=%v, end-seq=%v\n", + count, + st.state.lastACK+1, + st.seq, + ) + + total += int(count) + if total > N { + panic(fmt.Sprintf("Too many events acked (expected=%v, total=%v)", + N, total, + )) + } + + st.state.cb(int(count)) + st.state.lastACK = st.seq + st.state = nil + } +} + +func newBufferingEventLoop(b *Broker, size int, minEvents int, flushTimeout time.Duration) *bufferingEventLoop { + l := &bufferingEventLoop{ + broker: b, + maxEvents: size, + minEvents: minEvents, + flushTimeout: flushTimeout, + + events: b.events, + get: nil, + pubCancel: b.pubCancel, + acks: b.acks, + } + l.buf = newBatchBuffer(l.minEvents) + + l.timer = time.NewTimer(flushTimeout) + if !l.timer.Stop() { + <-l.timer.C + } + + return l +} + +func (l *bufferingEventLoop) run() { + var ( + broker = l.broker + ) + + for { + select { + case <-broker.done: + return + + case req := <-l.events: // producer pushing new event + l.handleInsert(&req) + + case req := <-l.pubCancel: // producer cancelling active events + l.handleCancel(&req) + + case req := <-l.get: // consumer asking for next batch + l.handleConsumer(&req) + + case l.schedACKS <- l.pendingACKs: + l.schedACKS = nil + l.pendingACKs = chanList{} + + case count := <-l.acks: + l.handleACK(count) + + case <-l.idleC: + l.idleC = nil + l.timer.Stop() + if l.buf.length() > 0 { + l.flushBuffer() + } + } + } +} + +func (l *bufferingEventLoop) handleInsert(req *pushRequest) { + if l.insert(req) { + l.eventCount++ + if l.eventCount == l.maxEvents { + l.events = nil // stop inserting events if upper limit is reached + } + + L := l.buf.length() + if !l.buf.flushed { + if L < l.minEvents { + l.startFlushTimer() + } else { + l.stopFlushTimer() + l.flushBuffer() + l.buf = newBatchBuffer(l.minEvents) + } + } else { + if L >= l.minEvents { + l.buf = newBatchBuffer(l.minEvents) + } + } + } +} + +func (l *bufferingEventLoop) insert(req *pushRequest) bool { + if req.state == nil { + l.buf.add(req.event, clientState{}) + return true + } + + st := req.state + if st.cancelled { + reportCancelledState(l.broker.logger, req) + return false + } + + l.buf.add(req.event, clientState{ + seq: req.seq, + state: st, + }) + return true +} + +func (l *bufferingEventLoop) handleCancel(req *producerCancelRequest) { + removed := 0 + if st := req.state; st != nil { + // remove from actively flushed buffers + for buf := l.flushList.head; buf != nil; buf = buf.next { + removed += buf.cancel(st) + } + if !l.buf.flushed { + removed += l.buf.cancel(st) + } + + st.cancelled = true + } + + if req.resp != nil { + req.resp <- producerCancelResponse{removed: removed} + } + + // remove flushed but empty buffers: + tmpList := flushList{} + for l.flushList.head != nil { + b := l.flushList.head + l.flushList.head = b.next + + if b.length() > 0 { + tmpList.add(b) + } + } + l.flushList = tmpList + if tmpList.empty() { + l.get = nil + } + + l.eventCount -= removed + if l.eventCount < l.maxEvents { + l.events = l.broker.events + } } -func (l *eventLoop) stopFlushTimer() { - l.forceFlush = false +func (l *bufferingEventLoop) handleConsumer(req *getRequest) { + buf := l.flushList.head + if buf == nil { + panic("get from non-flushed buffers") + } + + count := buf.length() + if count == 0 { + panic("empty buffer in flush list") + } + + if sz := req.sz; sz > 0 { + if sz < count { + count = sz + } + } + + if count == 0 { + panic("empty batch returned") + } + + events := buf.events[:count] + clients := buf.clients[:count] + ackChan := newACKChan(l.ackSeq, 0, count, clients) + l.ackSeq++ + + req.resp <- getResponse{ackChan, events} + l.pendingACKs.append(ackChan) + l.schedACKS = l.broker.scheduledACKs + + buf.events = buf.events[count:] + buf.clients = buf.clients[count:] + if buf.length() == 0 { + l.advanceFlushList() + } +} + +func (l *bufferingEventLoop) handleACK(count int) { + l.eventCount -= count + if l.eventCount < l.maxEvents { + l.events = l.broker.events + } +} + +func (l *bufferingEventLoop) startFlushTimer() { + if l.idleC == nil { + l.timer.Reset(l.flushTimeout) + l.idleC = l.timer.C + } +} + +func (l *bufferingEventLoop) stopFlushTimer() { if l.idleC != nil { l.idleC = nil if !l.timer.Stop() { @@ -175,9 +468,107 @@ func (l *eventLoop) stopFlushTimer() { } } -func (l *eventLoop) startFlushTimer() { - if l.idleC == nil && l.timer != nil { - l.timer.Reset(l.broker.idleTimeout) - l.idleC = l.timer.C +func (l *bufferingEventLoop) advanceFlushList() { + l.flushList.pop() + if l.flushList.count == 0 { + l.get = nil + + if l.buf.flushed { + l.buf = newBatchBuffer(l.minEvents) + } + } +} + +func (l *bufferingEventLoop) flushBuffer() { + l.buf.flushed = true + + if l.buf.length() == 0 { + panic("flushing empty buffer") } + + l.flushList.add(l.buf) + l.get = l.broker.requests +} + +func (l *bufferingEventLoop) processACK(lst chanList, N int) { + log := l.broker.logger + + total := 0 + lst.reverse() + for !lst.empty() { + current := lst.pop() + states := current.states + + for i := len(states) - 1; i >= 0; i-- { + st := &states[i] + if st.state == nil { + continue + } + + count := st.seq - st.state.lastACK + if count == 0 || count > math.MaxUint32/2 { + // seq number comparison did underflow. This happens only if st.seq has + // allready been acknowledged + // log.Debug("seq number already acked: ", st.seq) + + st.state = nil + continue + } + + log.Debugf("broker ACK events: count=%v, start-seq=%v, end-seq=%v\n", + count, + st.state.lastACK+1, + st.seq, + ) + + total += int(count) + if total > N { + panic(fmt.Sprintf("Too many events acked (expected=%v, total=%v)", + N, total, + )) + } + + st.state.cb(int(count)) + st.state.lastACK = st.seq + st.state = nil + } + } +} + +func (l *flushList) pop() { + l.count-- + if l.count > 0 { + l.head = l.head.next + } else { + l.head = nil + l.tail = nil + } +} + +func (l *flushList) empty() bool { + return l.head == nil +} + +func (l *flushList) add(b *batchBuffer) { + l.count++ + b.next = nil + if l.tail == nil { + l.head = b + l.tail = b + } else { + l.tail.next = b + l.tail = b + } +} + +func reportCancelledState(log logger, req *pushRequest) { + log.Debugf("cancelled producer - ignore event: %v\t%v\t%p", req.event, req.seq, req.state) + + // do not add waiting events if producer did send cancel signal + + st := req.state + if cb := st.dropCB; cb != nil { + cb(req.event.Content) + } + } diff --git a/libbeat/publisher/queue/memqueue/internal_api.go b/libbeat/publisher/queue/memqueue/internal_api.go index 211ae657fe9..13d7bbd61dc 100644 --- a/libbeat/publisher/queue/memqueue/internal_api.go +++ b/libbeat/publisher/queue/memqueue/internal_api.go @@ -27,10 +27,10 @@ type getRequest struct { } type getResponse struct { - buf []publisher.Event ack *ackChan + buf []publisher.Event } -type batchAckRequest struct{} +type batchAckMsg struct{} type batchCancelRequest struct{ ack *ackChan } diff --git a/libbeat/publisher/queue/memqueue/queue_test.go b/libbeat/publisher/queue/memqueue/queue_test.go index eda01937907..9da5461168b 100644 --- a/libbeat/publisher/queue/memqueue/queue_test.go +++ b/libbeat/publisher/queue/memqueue/queue_test.go @@ -21,7 +21,7 @@ func TestProduceConsumer(t *testing.T) { minEvents := 32 rand.Seed(seed) - events := rand.Intn(maxEvents-minEvents) + maxEvents + events := rand.Intn(maxEvents-minEvents) + minEvents batchSize := rand.Intn(events-8) + 4 bufferSize := rand.Intn(batchSize*2) + 4 @@ -34,22 +34,32 @@ func TestProduceConsumer(t *testing.T) { t.Log("batchSize: ", batchSize) t.Log("bufferSize: ", bufferSize) - factory := makeTestQueue(bufferSize) + testWith := func(factory queuetest.QueueFactory) func(t *testing.T) { + return func(t *testing.T) { + t.Run("single", func(t *testing.T) { + queuetest.TestSingleProducerConsumer(t, events, batchSize, factory) + }) + t.Run("multi", func(t *testing.T) { + queuetest.TestMultiProducerConsumer(t, events, batchSize, factory) + }) + } + } - t.Run("single", func(t *testing.T) { - queuetest.TestSingleProducerConsumer(t, events, batchSize, factory) - }) - t.Run("multi", func(t *testing.T) { - queuetest.TestMultiProducerConsumer(t, events, batchSize, factory) - }) + t.Run("direct", testWith(makeTestQueue(bufferSize, 0, 0))) + t.Run("flush", testWith(makeTestQueue(bufferSize, batchSize/2, 100*time.Millisecond))) } func TestProducerCancelRemovesEvents(t *testing.T) { - queuetest.TestProducerCancelRemovesEvents(t, makeTestQueue(1024)) + queuetest.TestProducerCancelRemovesEvents(t, makeTestQueue(1024, 0, 0)) } -func makeTestQueue(sz int) queuetest.QueueFactory { +func makeTestQueue(sz, minEvents int, flushTimeout time.Duration) queuetest.QueueFactory { return func() queue.Queue { - return NewBroker(Settings{Events: sz, WaitOnClose: true}) + return NewBroker(Settings{ + Events: sz, + FlushMinEvents: minEvents, + FlushTimeout: flushTimeout, + WaitOnClose: true, + }) } } diff --git a/libbeat/publisher/queue/memqueue/ringbuf.go b/libbeat/publisher/queue/memqueue/ringbuf.go new file mode 100644 index 00000000000..5f8b99037df --- /dev/null +++ b/libbeat/publisher/queue/memqueue/ringbuf.go @@ -0,0 +1,293 @@ +package memqueue + +import ( + "fmt" + + "github.com/elastic/beats/libbeat/publisher" +) + +// Internal event ring buffer. +// The ring is split into 2 regions. +// Region A contains active events to be send to consumers, while region B can +// only be filled by producers, if there is no more space in region A. Splitting +// the ring buffer into regions enables the broker to send batches of type +// []publisher.Event to the consumer without having to copy and/or grow/shrink the +// buffers. +type ringBuffer struct { + buf eventBuffer + + regA, regB region + reserved int // amount of events in region A actively processed/reserved +} + +type region struct { + index int + size int +} + +type eventBuffer struct { + logger logger + + events []publisher.Event + clients []clientState +} + +type clientState struct { + seq uint32 // event sequence number + state *produceState // the producer it's state used to compute and signal the ACK count +} + +func (b *eventBuffer) init(size int) { + b.events = make([]publisher.Event, size) + b.clients = make([]clientState, size) +} + +func (b *eventBuffer) Len() int { + return len(b.events) +} + +func (b *eventBuffer) Set(idx int, event publisher.Event, st clientState) { + // b.logger.Debugf("insert event: idx=%v, seq=%v\n", idx, st.seq) + + b.events[idx] = event + b.clients[idx] = st +} + +func newRingBuffer(log logger, size int) *ringBuffer { + b := &ringBuffer{} + b.init(log, size) + return b +} + +func (b *ringBuffer) init(log logger, size int) { + *b = ringBuffer{} + b.buf.init(size) + b.buf.logger = log +} + +func (b *ringBuffer) insert(event publisher.Event, client clientState) (bool, int) { + // log := b.buf.logger + // log.Debug("insert:") + // log.Debug(" region A:", b.regA) + // log.Debug(" region B:", b.regB) + // log.Debug(" reserved:", b.reserved) + // defer func() { + // log.Debug(" -> region A:", b.regA) + // log.Debug(" -> region B:", b.regB) + // log.Debug(" -> reserved:", b.reserved) + // }() + + // always insert into region B, if region B exists. + // That is, we have 2 regions and region A is currently processed by consumers + if b.regB.size > 0 { + // log.Debug(" - push into B region") + + idx := b.regB.index + b.regB.size + avail := b.regA.index - idx + if avail == 0 { + return false, 0 + } + + b.buf.Set(idx, event, client) + b.regB.size++ + + return true, avail - 1 + } + + // region B does not exist yet, check if region A is available for use + idx := b.regA.index + b.regA.size + // log.Debug(" - index: ", idx) + // log.Debug(" - buffer size: ", b.buf.Len()) + avail := b.buf.Len() - idx + if avail == 0 { // no more space in region A + // log.Debug(" - region A full") + + if b.regA.index == 0 { + // space to create region B, buffer is full + + // log.Debug(" - no space in region B") + + return false, 0 + } + + // create region B and insert events + // log.Debug(" - create region B") + b.regB.index = 0 + b.regB.size = 1 + b.buf.Set(0, event, client) + return true, b.regA.index - 1 + } + + // space available in region A -> let's append the event + // log.Debug(" - push into region A") + b.buf.Set(idx, event, client) + b.regA.size++ + return true, avail - 1 +} + +// cancel removes all buffered events matching `st`, not yet reserved by +// any consumer +func (b *ringBuffer) cancel(st *produceState) int { + // log := b.buf.logger + // log.Debug("cancel:") + // log.Debug(" region A:", b.regA) + // log.Debug(" region B:", b.regB) + // log.Debug(" reserved:", b.reserved) + // defer func() { + // log.Debug(" -> region A:", b.regA) + // log.Debug(" -> region B:", b.regB) + // log.Debug(" -> reserved:", b.reserved) + // }() + + // TODO: return if st has no pending events + + cancelB := b.cancelRegion(st, b.regB) + b.regB.size -= cancelB + + cancelA := b.cancelRegion(st, region{ + index: b.regA.index + b.reserved, + size: b.regA.size - b.reserved, + }) + b.regA.size -= cancelA + + return cancelA + cancelB +} + +func (b *ringBuffer) cancelRegion(st *produceState, reg region) (removed int) { + start := reg.index + end := start + reg.size + events := b.buf.events[start:end] + clients := b.buf.clients[start:end] + + toEvents := events[:0] + toClients := clients[:0] + + // filter loop + for i := 0; i < reg.size; i++ { + if clients[i].state == st { + continue // remove + } + + toEvents = append(toEvents, events[i]) + toClients = append(toClients, clients[i]) + } + + // re-initialize old buffer elements to help garbage collector + events = events[len(toEvents):] + clients = clients[len(toClients):] + for i := range events { + events[i] = publisher.Event{} + clients[i] = clientState{} + } + + return len(events) +} + +// activeBufferOffsets returns start and end offset +// of all available events in region A. +func (b *ringBuffer) activeBufferOffsets() (int, int) { + return b.regA.index, b.regA.index + b.regA.size +} + +// reserve returns up to `sz` events from the brokerBuffer, +// exclusively marking the events as 'reserved'. Subsequent calls to `reserve` +// will only return enqueued and non-reserved events from the buffer. +// If `sz == -1`, all available events will be reserved. +func (b *ringBuffer) reserve(sz int) (int, []publisher.Event) { + // log := b.buf.logger + // log.Debug("reserve: ", sz) + // log.Debug(" region A:", b.regA) + // log.Debug(" region B:", b.regB) + // log.Debug(" reserved:", b.reserved) + // defer func() { + // log.Debug(" -> region A:", b.regA) + // log.Debug(" -> region B:", b.regB) + // log.Debug(" -> reserved:", b.reserved) + // }() + + use := b.regA.size - b.reserved + // log.Debug(" - avail: ", use) + + if sz > 0 { + if use > sz { + use = sz + } + } + + start := b.regA.index + b.reserved + end := start + use + b.reserved += use + // log.Debug(" - start:", start) + // log.Debug(" - end:", end) + return start, b.buf.events[start:end] +} + +// ack up to sz events in region A +func (b *ringBuffer) ack(sz int) { + // log := b.buf.logger + // log.Debug("ack: ", sz) + // log.Debug(" region A:", b.regA) + // log.Debug(" region B:", b.regB) + // log.Debug(" reserved:", b.reserved) + // defer func() { + // log.Debug(" -> region A:", b.regA) + // log.Debug(" -> region B:", b.regB) + // log.Debug(" -> reserved:", b.reserved) + // }() + + if b.regA.size < sz { + panic(fmt.Errorf("Commit region to big (commit region=%v, buffer size=%v)", + sz, b.regA.size, + )) + } + + // clear region, so published events can be collected by the garbage collector: + end := b.regA.index + sz + for i := b.regA.index; i < end; i++ { + b.buf.events[i] = publisher.Event{} + } + + b.regA.index = end + b.regA.size -= sz + b.reserved -= sz + if b.regA.size == 0 { + // region A is empty, transfer region B into region A + b.regA = b.regB + b.regB.index = 0 + b.regB.size = 0 + } +} + +func (b *ringBuffer) Empty() bool { + return (b.regA.size - b.reserved) == 0 +} + +func (b *ringBuffer) Avail() int { + return b.regA.size - b.reserved +} + +func (b *ringBuffer) RegionBActive() bool { + return b.regB.size > 0 +} + +func (b *ringBuffer) RegionSizes() (int, int) { + return b.regA.size, b.regB.size +} + +func (b *ringBuffer) TotalAvail() int { + return b.regA.size + b.regB.size - b.reserved +} + +func (b *ringBuffer) Full() bool { + var avail int + if b.regB.size > 0 { + avail = b.regA.index - b.regB.index - b.regB.size + } else { + avail = b.buf.Len() - b.regA.index - b.regA.size + } + return avail == 0 +} + +func (b *ringBuffer) Size() int { + return b.buf.Len() +} From 7112a80ca65ccbf9b460e851ef95d4f5cd78c100 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 12 Sep 2017 16:03:08 +0200 Subject: [PATCH 18/96] Add support for post-update command (#5154) This command can be used in the beats to be run after `make update` is executed. This allows for example to modify the libbeat specific parts of the config file if needed. The command is optional in a beat, meaning if it does not exist it won't return an error. Also if it exist and it returns an error, make update will still skip the error. --- libbeat/scripts/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index d1291c9d777..db98d1088fd 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -293,6 +293,9 @@ endif @mkdir -p $(PWD)/_meta/kibana/default/index-pattern @${PYTHON_ENV}/bin/python ${ES_BEATS}/libbeat/scripts/generate_index_pattern.py --index '${BEAT_NAME}-*' --beat $(PWD) --version ${BEAT_VERSION} + # Runs the post-update target if it exists. Will not return errors + @-$(MAKE) post-update + .PHONY: docs docs: ## @build Builds the documents for the beat From 1530fec8a86ba37ca7c13e2c5d3f23c431dc87dc Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Tue, 12 Sep 2017 13:38:40 -0400 Subject: [PATCH 19/96] Fix kubernetes matcher registry lookup (#5159) --- CHANGELOG.asciidoc | 1 + libbeat/processors/add_kubernetes_metadata/registry.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index b4c94ff9474..9e530ecf748 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -13,6 +13,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di ==== Breaking changes *Affecting all Beats* +- Fix kubernetes matcher registry lookup. {pull}5159[5159] *Auditbeat* diff --git a/libbeat/processors/add_kubernetes_metadata/registry.go b/libbeat/processors/add_kubernetes_metadata/registry.go index 8ed886cab7c..4fd66e39bd4 100644 --- a/libbeat/processors/add_kubernetes_metadata/registry.go +++ b/libbeat/processors/add_kubernetes_metadata/registry.go @@ -34,7 +34,7 @@ func init() { p.MustRegisterLoader(indexerKey, func(ifc interface{}) error { i, ok := ifc.(indexerPlugin) if !ok { - return errors.New("plugin does not match output plugin type") + return errors.New("plugin does not match indexer plugin type") } name := i.name @@ -49,11 +49,11 @@ func init() { p.MustRegisterLoader(matcherKey, func(ifc interface{}) error { m, ok := ifc.(matcherPlugin) if !ok { - return errors.New("plugin does not match output plugin type") + return errors.New("plugin does not match matcher plugin type") } name := m.name - if Indexing.indexers[name] != nil { + if Indexing.matchers[name] != nil { return fmt.Errorf("matcher type %v already registered", name) } From 885c2d12ef1dc5be71fc3f3b8ea581d041a41ba1 Mon Sep 17 00:00:00 2001 From: Monica Sarbu Date: Tue, 12 Sep 2017 19:58:59 +0200 Subject: [PATCH 20/96] Fix the bullets from the redis docs (#5143) --- filebeat/docs/modules/redis.asciidoc | 4 ++-- filebeat/module/redis/_meta/docs.asciidoc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/filebeat/docs/modules/redis.asciidoc b/filebeat/docs/modules/redis.asciidoc index 757f1af9035..b1c10113628 100644 --- a/filebeat/docs/modules/redis.asciidoc +++ b/filebeat/docs/modules/redis.asciidoc @@ -9,8 +9,8 @@ experimental[] This module has two filesets: -* The `log` fileset collects and parses the logs that Redis writes to disk. The `slowlog` fileset -* connects to Redis via the network and retrieves the slow logs +* The `log` fileset collects and parses the logs that Redis writes to disk. +* The `slowlog` fileset connects to Redis via the network and retrieves the slow logs by using the `SLOWLOG` command. For the `log` fileset, make sure the `logfile` option is set in the Redis configuration file. For diff --git a/filebeat/module/redis/_meta/docs.asciidoc b/filebeat/module/redis/_meta/docs.asciidoc index ca65847158b..fda925d0ff6 100644 --- a/filebeat/module/redis/_meta/docs.asciidoc +++ b/filebeat/module/redis/_meta/docs.asciidoc @@ -4,8 +4,8 @@ experimental[] This module has two filesets: -* The `log` fileset collects and parses the logs that Redis writes to disk. The `slowlog` fileset -* connects to Redis via the network and retrieves the slow logs +* The `log` fileset collects and parses the logs that Redis writes to disk. +* The `slowlog` fileset connects to Redis via the network and retrieves the slow logs by using the `SLOWLOG` command. For the `log` fileset, make sure the `logfile` option is set in the Redis configuration file. For From 46dbdbd4f59e3c5beb68e2f52a8e72df91b83d3e Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Tue, 12 Sep 2017 21:07:28 +0200 Subject: [PATCH 21/96] Add @metadata.version to json events (#5166) * Add @metadata.version to json events - pass beat.Info into the codecs - update json encoder to include @metadata.version (for Console, File, Logstash, Redis, Kafka) * Update Ls docs to use the version field * Fix test build * s/second/third/ * Fix metricbeat unit tests --- CHANGELOG.asciidoc | 1 + libbeat/docs/gettingstarted.asciidoc | 5 ++-- libbeat/docs/outputconfig.asciidoc | 17 ++++++----- .../report/elasticsearch/elasticsearch.go | 1 + libbeat/outputs/codec/codec_reg.go | 7 +++-- libbeat/outputs/codec/format/format.go | 2 +- libbeat/outputs/codec/json/event.go | 16 +++++----- libbeat/outputs/codec/json/json.go | 21 ++++++++----- libbeat/outputs/codec/json/json_test.go | 9 +++--- libbeat/outputs/console/console.go | 4 +-- libbeat/outputs/console/console_test.go | 8 ++--- libbeat/outputs/fileout/file.go | 6 ++-- libbeat/outputs/kafka/kafka.go | 2 +- libbeat/outputs/logstash/async.go | 4 ++- libbeat/outputs/logstash/async_test.go | 3 +- libbeat/outputs/logstash/enc.go | 4 +-- libbeat/outputs/logstash/event.go | 30 ------------------- libbeat/outputs/logstash/logstash.go | 4 +-- libbeat/outputs/logstash/sync.go | 4 ++- libbeat/outputs/logstash/sync_test.go | 3 +- libbeat/outputs/redis/redis.go | 2 +- libbeat/publisher/pipeline/module.go | 2 +- libbeat/publisher/pipeline/pipeline.go | 4 +++ libbeat/publisher/pipeline/processor.go | 8 ++--- metricbeat/mb/module/example_test.go | 5 ++-- metricbeat/mb/testing/data_generator.go | 2 +- 26 files changed, 84 insertions(+), 90 deletions(-) delete mode 100644 libbeat/outputs/logstash/event.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 9e530ecf748..38de277fa3c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -69,6 +69,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Changed the hashbang used in the beat helper script from `/bin/bash` to `/usr/bin/env bash`. {pull}5051[5051] - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051] - Fix reloader error message to only print on actual error {pull}5066[5066] +- Add @metadata.version to events send to Logstash. {pull}5166[5166] *Auditbeat* diff --git a/libbeat/docs/gettingstarted.asciidoc b/libbeat/docs/gettingstarted.asciidoc index 002c0adf4f8..df5db066c53 100644 --- a/libbeat/docs/gettingstarted.asciidoc +++ b/libbeat/docs/gettingstarted.asciidoc @@ -296,13 +296,14 @@ output { elasticsearch { hosts => "localhost:9200" manage_template => false - index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" <1> + index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" <1> document_type => "%{[@metadata][type]}" <2> } } ------------------------------------------------------------------------------ <1> `%{[@metadata][beat]}` sets the first part of the index name to the value -of the `beat` metadata field, and `%{+YYYY.MM.dd}` sets the second part of the +of the `beat` metadata field, `%{[@metadata][version]}` sets the second part of +the name to the beat's version, and `%{+YYYY.MM.dd}` sets the third part of the name to a date based on the Logstash `@timestamp` field. For example: +{beatname_lc}-2017.03.29+. <2> `%{[@metadata][type]}` sets the document type based on the value of the `type` diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index dc71dc25cab..3578f503f5b 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -322,18 +322,18 @@ use in Logstash for indexing and filtering: ... "@metadata": { <1> "beat": "{beatname_lc}", <2> - "type": "doc" <3> + "version": "{stack-version}" <3> + "type": "doc" <4> } } ------------------------------------------------------------------------------ -<1> {beatname_uc} uses the `@metadata` field to send metadata to Logstash. The -contents of the `@metadata` field only exist in Logstash and are not part of any -events sent from Logstash. See the +<1> {beatname_uc} uses the `@metadata` field to send metadata to Logstash. See the {logstashdoc}/event-dependent-configuration.html#metadata[Logstash documentation] for more about the `@metadata` field. <2> The default is {beatname_lc}. To change this value, set the <> option in the {beatname_uc} config file. -<3> The value of `type` is currently hardcoded to `doc`. It was used by previous +<3> The beats current version. +<4> The value of `type` is currently hardcoded to `doc`. It was used by previous Logstash configs to set the type of the document in Elasticsearch. @@ -359,14 +359,15 @@ input { output { elasticsearch { hosts => ["http://localhost:9200"] - index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" <1> + index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" <1> } } ------------------------------------------------------------------------------ <1> `%{[@metadata][beat]}` sets the first part of the index name to the value -of the `beat` metadata field, and `%{+YYYY.MM.dd}` sets the second part of the +of the `beat` metadata field, `%{[@metadata][version]}` sets the second part to +the beat's version, and `%{+YYYY.MM.dd}` sets the third part of the name to a date based on the Logstash `@timestamp` field. For example: -+{beatname_lc}-2017.03.29+. ++{beatname_lc}-{stack-version}-2017.03.29+. Events indexed into Elasticsearch with the Logstash configuration shown here will be similar to events directly indexed by Beats into Elasticsearch. diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index 8383e9e4fa8..dab27f87fb7 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -114,6 +114,7 @@ func makeReporter(beat beat.Info, cfg *common.Config) (report.Reporter, error) { monitoring := monitoring.Default.NewRegistry("xpack.monitoring") pipeline, err := pipeline.New( + beat, monitoring, queueFactory, out, pipeline.Settings{ WaitClose: 0, diff --git a/libbeat/outputs/codec/codec_reg.go b/libbeat/outputs/codec/codec_reg.go index fa5d0ee4e71..3512db3dd4c 100644 --- a/libbeat/outputs/codec/codec_reg.go +++ b/libbeat/outputs/codec/codec_reg.go @@ -3,10 +3,11 @@ package codec import ( "fmt" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" ) -type Factory func(*common.Config) (Codec, error) +type Factory func(beat.Info, *common.Config) (Codec, error) type Config struct { Namespace common.ConfigNamespace `config:",inline"` @@ -21,7 +22,7 @@ func RegisterType(name string, gen Factory) { codecs[name] = gen } -func CreateEncoder(cfg Config) (Codec, error) { +func CreateEncoder(info beat.Info, cfg Config) (Codec, error) { // default to json codec codec := "json" if name := cfg.Namespace.Name(); name != "" { @@ -32,5 +33,5 @@ func CreateEncoder(cfg Config) (Codec, error) { if factory == nil { return nil, fmt.Errorf("'%v' output codec is not available", codec) } - return factory(cfg.Namespace.Config()) + return factory(info, cfg.Namespace.Config()) } diff --git a/libbeat/outputs/codec/format/format.go b/libbeat/outputs/codec/format/format.go index ab346ba9af3..e26bc269c59 100644 --- a/libbeat/outputs/codec/format/format.go +++ b/libbeat/outputs/codec/format/format.go @@ -18,7 +18,7 @@ type Config struct { } func init() { - codec.RegisterType("format", func(cfg *common.Config) (codec.Codec, error) { + codec.RegisterType("format", func(_ beat.Info, cfg *common.Config) (codec.Codec, error) { config := Config{} if cfg == nil { return nil, errors.New("empty format codec configuration") diff --git a/libbeat/outputs/codec/json/event.go b/libbeat/outputs/codec/json/event.go index 417022630c8..e6905d352c0 100644 --- a/libbeat/outputs/codec/json/event.go +++ b/libbeat/outputs/codec/json/event.go @@ -17,18 +17,20 @@ type event struct { // Meta defines common event metadata to be stored in '@metadata' type meta struct { - Beat string `struct:"beat"` - Type string `struct:"type"` - Fields map[string]interface{} `struct:",inline"` + Beat string `struct:"beat"` + Type string `struct:"type"` + Version string `struct:"version"` + Fields map[string]interface{} `struct:",inline"` } -func makeEvent(index string, in *beat.Event) event { +func makeEvent(index, version string, in *beat.Event) event { return event{ Timestamp: in.Timestamp, Meta: meta{ - Beat: index, - Type: "doc", - Fields: in.Meta, + Beat: index, + Version: version, + Type: "doc", + Fields: in.Meta, }, Fields: in.Fields, } diff --git a/libbeat/outputs/codec/json/json.go b/libbeat/outputs/codec/json/json.go index ea74a024305..a016cf10b70 100644 --- a/libbeat/outputs/codec/json/json.go +++ b/libbeat/outputs/codec/json/json.go @@ -12,10 +12,12 @@ import ( "github.com/elastic/beats/libbeat/outputs/codec" ) +// Encoder for serializing a beat.Event to json. type Encoder struct { - buf bytes.Buffer - folder *gotype.Iterator - pretty bool + buf bytes.Buffer + folder *gotype.Iterator + pretty bool + version string } type config struct { @@ -27,7 +29,7 @@ var defaultConfig = config{ } func init() { - codec.RegisterType("json", func(cfg *common.Config) (codec.Codec, error) { + codec.RegisterType("json", func(info beat.Info, cfg *common.Config) (codec.Codec, error) { config := defaultConfig if cfg != nil { if err := cfg.Unpack(&config); err != nil { @@ -35,12 +37,13 @@ func init() { } } - return New(config.Pretty), nil + return New(config.Pretty, info.Version), nil }) } -func New(pretty bool) *Encoder { - e := &Encoder{pretty: pretty} +// New creates a new json Encoder. +func New(pretty bool, version string) *Encoder { + e := &Encoder{pretty: pretty, version: version} e.reset() return e } @@ -62,9 +65,11 @@ func (e *Encoder) reset() { } } +// Encode serializies a beat event to JSON. It adds additional metadata in the +// `@metadata` namespace. func (e *Encoder) Encode(index string, event *beat.Event) ([]byte, error) { e.buf.Reset() - err := e.folder.Fold(makeEvent(index, event)) + err := e.folder.Fold(makeEvent(index, e.version, event)) if err != nil { e.reset() return nil, err diff --git a/libbeat/outputs/codec/json/json_test.go b/libbeat/outputs/codec/json/json_test.go index f8f1f97b656..3e9d07186ca 100644 --- a/libbeat/outputs/codec/json/json_test.go +++ b/libbeat/outputs/codec/json/json_test.go @@ -8,9 +8,9 @@ import ( ) func TestJsonCodec(t *testing.T) { - expectedValue := `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"doc"},"msg":"message"}` + expectedValue := `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"doc","version":"1.2.3"},"msg":"message"}` - codec := New(false) + codec := New(false, "1.2.3") output, err := codec.Encode("test", &beat.Event{Fields: common.MapStr{"msg": "message"}}) if err != nil { @@ -27,12 +27,13 @@ func TestJsonWriterPrettyPrint(t *testing.T) { "@timestamp": "0001-01-01T00:00:00.000Z", "@metadata": { "beat": "test", - "type": "doc" + "type": "doc", + "version": "1.2.3" }, "msg": "message" }` - codec := New(true) + codec := New(true, "1.2.3") output, err := codec.Encode("test", &beat.Event{Fields: common.MapStr{"msg": "message"}}) if err != nil { diff --git a/libbeat/outputs/console/console.go b/libbeat/outputs/console/console.go index 9ef7e84a90c..605121535fc 100644 --- a/libbeat/outputs/console/console.go +++ b/libbeat/outputs/console/console.go @@ -48,12 +48,12 @@ func makeConsole( var enc codec.Codec if config.Codec.Namespace.IsSet() { - enc, err = codec.CreateEncoder(config.Codec) + enc, err = codec.CreateEncoder(beat, config.Codec) if err != nil { return outputs.Fail(err) } } else { - enc = json.New(config.Pretty) + enc = json.New(config.Pretty, beat.Version) } index := beat.Beat diff --git a/libbeat/outputs/console/console_test.go b/libbeat/outputs/console/console_test.go index 6ba0c17dbd7..92bebd619a8 100644 --- a/libbeat/outputs/console/console_test.go +++ b/libbeat/outputs/console/console_test.go @@ -60,19 +60,19 @@ func TestConsoleOutput(t *testing.T) { }{ { "single json event (pretty=false)", - json.New(false), + json.New(false, "1.2.3"), []beat.Event{ {Fields: event("field", "value")}, }, - "{\"@timestamp\":\"0001-01-01T00:00:00.000Z\",\"@metadata\":{\"beat\":\"test\",\"type\":\"doc\"},\"field\":\"value\"}\n", + "{\"@timestamp\":\"0001-01-01T00:00:00.000Z\",\"@metadata\":{\"beat\":\"test\",\"type\":\"doc\",\"version\":\"1.2.3\"},\"field\":\"value\"}\n", }, { "single json event (pretty=true)", - json.New(true), + json.New(true, "1.2.3"), []beat.Event{ {Fields: event("field", "value")}, }, - "{\n \"@timestamp\": \"0001-01-01T00:00:00.000Z\",\n \"@metadata\": {\n \"beat\": \"test\",\n \"type\": \"doc\"\n },\n \"field\": \"value\"\n}\n", + "{\n \"@timestamp\": \"0001-01-01T00:00:00.000Z\",\n \"@metadata\": {\n \"beat\": \"test\",\n \"type\": \"doc\",\n \"version\": \"1.2.3\"\n },\n \"field\": \"value\"\n}\n", }, // TODO: enable test after update fmtstr support to beat.Event { diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index faa9cc79d7c..bf569058f39 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -35,14 +35,14 @@ func makeFileout( cfg.SetInt("bulk_max_size", -1, -1) fo := &fileOutput{beat: beat, stats: stats} - if err := fo.init(config); err != nil { + if err := fo.init(beat, config); err != nil { return outputs.Fail(err) } return outputs.Success(-1, 0, fo) } -func (out *fileOutput) init(config config) error { +func (out *fileOutput) init(beat beat.Info, config config) error { var err error out.rotator.Path = config.Path @@ -51,7 +51,7 @@ func (out *fileOutput) init(config config) error { out.rotator.Name = out.beat.Beat } - enc, err := codec.CreateEncoder(config.Codec) + enc, err := codec.CreateEncoder(beat, config.Codec) if err != nil { return err } diff --git a/libbeat/outputs/kafka/kafka.go b/libbeat/outputs/kafka/kafka.go index c7d2cc21bef..9896e72ef4a 100644 --- a/libbeat/outputs/kafka/kafka.go +++ b/libbeat/outputs/kafka/kafka.go @@ -143,7 +143,7 @@ func makeKafka( return outputs.Fail(err) } - codec, err := codec.CreateEncoder(config.Codec) + codec, err := codec.CreateEncoder(beat, config.Codec) if err != nil { return outputs.Fail(err) } diff --git a/libbeat/outputs/logstash/async.go b/libbeat/outputs/logstash/async.go index 10c0ed1fe23..70eb6609f58 100644 --- a/libbeat/outputs/logstash/async.go +++ b/libbeat/outputs/logstash/async.go @@ -4,6 +4,7 @@ import ( "net" "time" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common/atomic" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/outputs" @@ -32,6 +33,7 @@ type msgRef struct { } func newAsyncClient( + beat beat.Info, conn *transport.Client, stats *outputs.Stats, config *Config, @@ -49,7 +51,7 @@ func newAsyncClient( logp.Warn(`The async Logstash client does not support the "ttl" option`) } - enc := makeLogstashEventEncoder(config.Index) + enc := makeLogstashEventEncoder(beat, config.Index) queueSize := config.Pipelining - 1 timeout := config.Timeout diff --git a/libbeat/outputs/logstash/async_test.go b/libbeat/outputs/logstash/async_test.go index 4964b187599..2bd94e244f3 100644 --- a/libbeat/outputs/logstash/async_test.go +++ b/libbeat/outputs/logstash/async_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/outputs" "github.com/elastic/beats/libbeat/outputs/outest" "github.com/elastic/beats/libbeat/outputs/transport" @@ -35,7 +36,7 @@ func makeAsyncTestClient(conn *transport.Client) testClientDriver { config := defaultConfig config.Timeout = 1 * time.Second config.Pipelining = 3 - client, err := newAsyncClient(conn, nil, &config) + client, err := newAsyncClient(beat.Info{}, conn, nil, &config) if err != nil { panic(err) } diff --git a/libbeat/outputs/logstash/enc.go b/libbeat/outputs/logstash/enc.go index 28969ad184c..63963300655 100644 --- a/libbeat/outputs/logstash/enc.go +++ b/libbeat/outputs/logstash/enc.go @@ -5,8 +5,8 @@ import ( "github.com/elastic/beats/libbeat/outputs/codec/json" ) -func makeLogstashEventEncoder(index string) func(interface{}) ([]byte, error) { - enc := json.New(false) +func makeLogstashEventEncoder(info beat.Info, index string) func(interface{}) ([]byte, error) { + enc := json.New(false, info.Version) return func(event interface{}) ([]byte, error) { return enc.Encode(index, event.(*beat.Event)) } diff --git a/libbeat/outputs/logstash/event.go b/libbeat/outputs/logstash/event.go deleted file mode 100644 index ce223fb6c5b..00000000000 --- a/libbeat/outputs/logstash/event.go +++ /dev/null @@ -1,30 +0,0 @@ -package logstash - -/* -// Event describes the event strucutre for events -// (in-)directly send to logstash -type Event struct { - Timestamp time.Time `struct:"@timestamp"` - Meta Meta `struct:"@metadata"` - Fields common.MapStr `struct:",inline"` -} - -// Meta defines common event metadata to be stored in '@metadata' -type Meta struct { - Beat string `struct:"beat"` - Type string `struct:"type"` - Fields map[string]interface{} `struct:",inline"` -} - -func MakeEvent(index string, event *beat.Event) Event { - return Event{ - Timestamp: event.Timestamp, - Meta: Meta{ - Beat: index, - Type: "doc", - Fields: event.Meta, - }, - Fields: event.Fields, - } -} -*/ diff --git a/libbeat/outputs/logstash/logstash.go b/libbeat/outputs/logstash/logstash.go index d8261586e82..6e519f543b3 100644 --- a/libbeat/outputs/logstash/logstash.go +++ b/libbeat/outputs/logstash/logstash.go @@ -60,9 +60,9 @@ func makeLogstash( } if config.Pipelining > 0 { - client, err = newAsyncClient(conn, stats, config) + client, err = newAsyncClient(beat, conn, stats, config) } else { - client, err = newSyncClient(conn, stats, config) + client, err = newSyncClient(beat, conn, stats, config) } if err != nil { return outputs.Fail(err) diff --git a/libbeat/outputs/logstash/sync.go b/libbeat/outputs/logstash/sync.go index 4bf2bdbd98b..08bf181b776 100644 --- a/libbeat/outputs/logstash/sync.go +++ b/libbeat/outputs/logstash/sync.go @@ -3,6 +3,7 @@ package logstash import ( "time" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/outputs" "github.com/elastic/beats/libbeat/outputs/transport" @@ -20,6 +21,7 @@ type syncClient struct { } func newSyncClient( + beat beat.Info, conn *transport.Client, stats *outputs.Stats, config *Config, @@ -38,7 +40,7 @@ func newSyncClient( } var err error - enc := makeLogstashEventEncoder(config.Index) + enc := makeLogstashEventEncoder(beat, config.Index) c.client, err = v2.NewSyncClientWithConn(conn, v2.JSONEncoder(enc), v2.Timeout(config.Timeout), diff --git a/libbeat/outputs/logstash/sync_test.go b/libbeat/outputs/logstash/sync_test.go index 932aa484ceb..ab89c3e56a7 100644 --- a/libbeat/outputs/logstash/sync_test.go +++ b/libbeat/outputs/logstash/sync_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/outputs" "github.com/elastic/beats/libbeat/outputs/outest" "github.com/elastic/beats/libbeat/outputs/transport" @@ -48,7 +49,7 @@ func makeTestClient(conn *transport.Client) testClientDriver { config := defaultConfig config.Timeout = 1 * time.Second config.TTL = 5 * time.Second - client, err := newSyncClient(conn, nil, &config) + client, err := newSyncClient(beat.Info{}, conn, nil, &config) if err != nil { panic(err) } diff --git a/libbeat/outputs/redis/redis.go b/libbeat/outputs/redis/redis.go index 2277d6dce27..53df2c83427 100644 --- a/libbeat/outputs/redis/redis.go +++ b/libbeat/outputs/redis/redis.go @@ -94,7 +94,7 @@ func makeRedis( clients := make([]outputs.NetworkClient, len(hosts)) for i, host := range hosts { - enc, err := codec.CreateEncoder(config.Codec) + enc, err := codec.CreateEncoder(beat, config.Codec) if err != nil { return outputs.Fail(err) } diff --git a/libbeat/publisher/pipeline/module.go b/libbeat/publisher/pipeline/module.go index 769d7a0a536..302d4fa8d8d 100644 --- a/libbeat/publisher/pipeline/module.go +++ b/libbeat/publisher/pipeline/module.go @@ -72,7 +72,7 @@ func Load( return nil, err } - p, err := New(reg, queueBuilder, out, settings) + p, err := New(beatInfo, reg, queueBuilder, out, settings) if err != nil { return nil, err } diff --git a/libbeat/publisher/pipeline/pipeline.go b/libbeat/publisher/pipeline/pipeline.go index 9c88d22fe34..c6cb740a60d 100644 --- a/libbeat/publisher/pipeline/pipeline.go +++ b/libbeat/publisher/pipeline/pipeline.go @@ -33,6 +33,8 @@ import ( // Processors in the pipeline are executed in the clients go-routine, before // entering the queue. No filtering/processing will occur on the output side. type Pipeline struct { + beatInfo beat.Info + logger *logp.Logger queue queue.Queue output *outputController @@ -130,6 +132,7 @@ type queueFactory func(queue.Eventer) (queue.Queue, error) // The new pipeline will take ownership of queue and outputs. On Close, the // queue and outputs will be closed. func New( + beat beat.Info, metrics *monitoring.Registry, queueFactory queueFactory, out outputs.Group, @@ -142,6 +145,7 @@ func New( processors := settings.Processors disabledOutput := settings.Disabled p := &Pipeline{ + beatInfo: beat, logger: log, observer: nilObserver, waitCloseMode: settings.WaitCloseMode, diff --git a/libbeat/publisher/pipeline/processor.go b/libbeat/publisher/pipeline/processor.go index 209150f5821..2c8d3f245fa 100644 --- a/libbeat/publisher/pipeline/processor.go +++ b/libbeat/publisher/pipeline/processor.go @@ -95,7 +95,7 @@ func (p *Pipeline) newProcessorPipeline( // setup 9: debug print final event (P) if logp.IsDebug("publish") { - processors.add(debugPrintProcessor()) + processors.add(debugPrintProcessor(p.beatInfo)) } // setup 10: drop all events if outputs are disabled (P) @@ -244,17 +244,17 @@ func makeAddFieldsProcessor(name string, fields common.MapStr, copy bool) *proce return newAnnotateProcessor(name, fn) } -func debugPrintProcessor() *processorFn { +func debugPrintProcessor(info beat.Info) *processorFn { // ensure only one go-routine is using the encoder (in case // beat.Client is shared between multiple go-routines by accident) var mux sync.Mutex - encoder := json.New(true) + encoder := json.New(true, info.Version) return newProcessor("debugPrint", func(event *beat.Event) (*beat.Event, error) { mux.Lock() defer mux.Unlock() - b, err := encoder.Encode("", event) + b, err := encoder.Encode(info.Beat, event) if err != nil { return event, nil } diff --git a/metricbeat/mb/module/example_test.go b/metricbeat/mb/module/example_test.go index c90687eff32..750c48f3a5f 100644 --- a/metricbeat/mb/module/example_test.go +++ b/metricbeat/mb/module/example_test.go @@ -70,7 +70,8 @@ func ExampleWrapper() { // { // "@metadata": { // "beat": "noindex", - // "type": "doc" + // "type": "doc", + // "version": "1.2.3" // }, // "@timestamp": "2016-05-10T23:27:58.485Z", // "fake": { @@ -130,7 +131,7 @@ func ExampleRunner() { } func encodeEvent(event beat.Event) (string, error) { - output, err := json.New(false).Encode("noindex", &event) + output, err := json.New(false, "1.2.3").Encode("noindex", &event) if err != nil { return "", nil } diff --git a/metricbeat/mb/testing/data_generator.go b/metricbeat/mb/testing/data_generator.go index 77ad9922c84..3d01260f186 100644 --- a/metricbeat/mb/testing/data_generator.go +++ b/metricbeat/mb/testing/data_generator.go @@ -102,7 +102,7 @@ func WriteEventToDataJSON(t testing.TB, fullEvent beat.Event) { } // use json output codec to encode event to json - output, err := json.New(true).Encode("noindex", &fullEvent) + output, err := json.New(true, "1.2.3").Encode("noindex", &fullEvent) if err != nil { t.Fatal(err) } From 052172f5be2264df940b1062bc9d69add2431183 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 13 Sep 2017 10:21:57 +0200 Subject: [PATCH 22/96] Update queue.mem defaults (#5150) * Update queue.mem defaults - Enable flush based memory queue by default: Increases chance of pushing bigger batches of events to the outputs. Especially if number of CPUs is limited to 1 or 2 this helps with throughput, as outputs and producers go-routines will interleave less. - Set flush.events=2048 and flush.timeout=1s * set queue flush parameters in system tests update queue flush parameters to min_events of 8 and a timeout of 0.1s. This hopefully stabilizes some time sensitive tests. * Fix typo in CHANGELOG --- CHANGELOG.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 4 ++-- auditbeat/tests/system/config/auditbeat.yml.j2 | 5 +++++ filebeat/filebeat.reference.yml | 4 ++-- filebeat/tests/system/config/filebeat.yml.j2 | 7 +++++++ heartbeat/heartbeat.reference.yml | 4 ++-- heartbeat/tests/system/config/heartbeat.yml.j2 | 9 +++++++++ libbeat/_meta/config.reference.yml | 4 ++-- libbeat/publisher/queue/memqueue/config.go | 4 ++-- libbeat/tests/system/config/mockbeat.yml.j2 | 6 ++++++ metricbeat/metricbeat.reference.yml | 4 ++-- metricbeat/tests/system/config/metricbeat.yml.j2 | 7 +++++++ packetbeat/packetbeat.reference.yml | 4 ++-- packetbeat/tests/system/config/packetbeat.yml.j2 | 7 +++++++ winlogbeat/tests/system/config/winlogbeat.yml.j2 | 7 +++++++ winlogbeat/winlogbeat.reference.yml | 4 ++-- 16 files changed, 65 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 38de277fa3c..4405dc482f8 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -69,6 +69,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Changed the hashbang used in the beat helper script from `/bin/bash` to `/usr/bin/env bash`. {pull}5051[5051] - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051] - Fix reloader error message to only print on actual error {pull}5066[5066] +- Enable flush timeout by default. {pull}5150[5150] - Add @metadata.version to events send to Logstash. {pull}5166[5166] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 7ad40e0c424..6e37829e218 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -101,11 +101,11 @@ auditbeat.modules: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/auditbeat/tests/system/config/auditbeat.yml.j2 b/auditbeat/tests/system/config/auditbeat.yml.j2 index 21911d7d039..0cdbf5bbf84 100644 --- a/auditbeat/tests/system/config/auditbeat.yml.j2 +++ b/auditbeat/tests/system/config/auditbeat.yml.j2 @@ -17,6 +17,11 @@ auditbeat.modules: {% endif -%} {%- endfor %} +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + output.file: path: '{{ output_file_path|default(beat.working_dir + "/output") }}' filename: {{ output_file_filename|default("auditbeat") }} diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 11f0fe833be..05c76a7e7b6 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -521,11 +521,11 @@ filebeat.prospectors: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index a41e1db935f..9a7330b35a9 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -156,6 +156,13 @@ processors: {%- endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index ac73a44c4f1..23350d2ac1c 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -250,11 +250,11 @@ heartbeat.scheduler: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/heartbeat/tests/system/config/heartbeat.yml.j2 b/heartbeat/tests/system/config/heartbeat.yml.j2 index 664715c60b6..42407d56841 100644 --- a/heartbeat/tests/system/config/heartbeat.yml.j2 +++ b/heartbeat/tests/system/config/heartbeat.yml.j2 @@ -56,6 +56,15 @@ fields: {% endfor -%} {% endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + +#================================ Outputs ===================================== + output.file: path: {{ output_file_path|default(beat.working_dir + "/output") }} filename: "{{ output_file_filename|default("heartbeat") }}" diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 8f69c25573d..8248a2ead01 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -36,11 +36,11 @@ # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/libbeat/publisher/queue/memqueue/config.go b/libbeat/publisher/queue/memqueue/config.go index ef843484a37..54da7aeb8f1 100644 --- a/libbeat/publisher/queue/memqueue/config.go +++ b/libbeat/publisher/queue/memqueue/config.go @@ -13,8 +13,8 @@ type config struct { var defaultConfig = config{ Events: 4 * 1024, - FlushMinEvents: 0, - FlushTimeout: 0, + FlushMinEvents: 2 * 1024, + FlushTimeout: 1 * time.Second, } func (c *config) Validate() error { diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index 03909dfb3ac..a0415a953d2 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -18,6 +18,12 @@ tags: [ {%- endfor -%} {%- endif -%}] +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s ############################# Output ############################################ diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 0a9a09b6fa1..6356dd3d70a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -480,11 +480,11 @@ metricbeat.modules: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index 519bf3ef1fe..2c02f4b78c9 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -139,6 +139,13 @@ processors: {%- endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 633549190e5..262cbce5c19 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -488,11 +488,11 @@ packetbeat.protocols: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 354958c38d3..2523d2ea69f 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -193,6 +193,13 @@ processors: {%- endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/winlogbeat/tests/system/config/winlogbeat.yml.j2 b/winlogbeat/tests/system/config/winlogbeat.yml.j2 index f221376cbff..8ca238a13ba 100644 --- a/winlogbeat/tests/system/config/winlogbeat.yml.j2 +++ b/winlogbeat/tests/system/config/winlogbeat.yml.j2 @@ -77,6 +77,13 @@ fields: {% endfor -%} {% endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 6d2a0570081..96ba9830c88 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -65,11 +65,11 @@ winlogbeat.event_logs: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. From 5d26d2c2327f27efe22a675976c9300415278909 Mon Sep 17 00:00:00 2001 From: Mark Janssen Date: Fri, 15 Sep 2017 10:18:50 +0200 Subject: [PATCH 23/96] Fix systemd unit file name in Debian package (#5174) Fixes duplicate daemon start after #5086. --- dev-tools/packer/platforms/debian/run.sh.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-tools/packer/platforms/debian/run.sh.j2 b/dev-tools/packer/platforms/debian/run.sh.j2 index 6eeb232f599..61c2e89e844 100644 --- a/dev-tools/packer/platforms/debian/run.sh.j2 +++ b/dev-tools/packer/platforms/debian/run.sh.j2 @@ -4,8 +4,9 @@ set -e cd /build -# the init scripts needs to have the right name +# the init script and systemd unit file need to have the right name cp ${RUNID}.init /tmp/{{.beat_pkg_name}}.init +cp ${RUNID}.service /tmp/{{.beat_pkg_name}}.service # add SNAPSHOT if it was requested VERSION="{{.version}}" @@ -23,7 +24,7 @@ FPM_ARGS=( --description "{{.beat_description}}" --url {{.beat_url}} --deb-init /tmp/{{.beat_pkg_name}}.init - --deb-systemd ${RUNID}.service + --deb-systemd /tmp/{{.beat_pkg_name}}.service --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml homedir/=/usr/share/{{.beat_name}} beatname-${RUNID}.sh=/usr/bin/{{.beat_name}} From dc5ab42a144c99a15f17f41388a1cf331d2141e2 Mon Sep 17 00:00:00 2001 From: Phillip Baker Date: Fri, 15 Sep 2017 07:41:57 -0400 Subject: [PATCH 24/96] Add cloudwatchmetric beat to list of community beats. (#5177) --- libbeat/docs/communitybeats.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/docs/communitybeats.asciidoc b/libbeat/docs/communitybeats.asciidoc index 4b130a6f868..6ed0aeff937 100644 --- a/libbeat/docs/communitybeats.asciidoc +++ b/libbeat/docs/communitybeats.asciidoc @@ -27,6 +27,7 @@ https://github.com/goomzee/cassandrabeat[cassandrabeat]:: Uses Cassandra's nodet https://github.com/hartfordfive/cloudflarebeat[cloudflarebeat]:: Indexes log entries from the Cloudflare Enterprise Log Share API. https://github.com/jarl-tornroos/cloudfrontbeat[cloudfrontbeat]:: Reads log events from Amazon Web Services https://aws.amazon.com/cloudfront/[CloudFront]. https://github.com/aidan-/cloudtrailbeat[cloudtrailbeat]:: Reads events from Amazon Web Services' https://aws.amazon.com/cloudtrail/[CloudTrail]. +https://github.com/narmitech/cloudwatchmetricbeat[cloudwatchmetricbeat]:: A beat for Amazon Web Services' https://aws.amazon.com/cloudwatch/details/#other-aws-resource-monitoring[CloudWatch Metrics]. https://github.com/e-travel/cloudwatchlogsbeat[cloudwatchlogsbeat]:: Reads log events from Amazon Web Services' https://aws.amazon.com/cloudwatch/details/#log-monitoring[CloudWatch Logs]. https://github.com/raboof/connbeat[connbeat]:: Exposes metadata about TCP connections. https://github.com/Pravoru/consulbeat[consulbeat]:: Reads services health checks from consul and pushes them to Elastic. From 9a4fbbb7a726a99caf6fdc11fab732e8ad24f2ab Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 18 Sep 2017 10:27:34 +0200 Subject: [PATCH 25/96] Add open_pr script (#5186) This is a convenience script for opening PRs without messing as much with the Github web UI. For example, it allows to quickly create "close changelog" PRs against a particular branch. * Add --labels flag --- dev-tools/cherrypick_pr | 2 +- dev-tools/open_pr | 102 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100755 dev-tools/open_pr diff --git a/dev-tools/cherrypick_pr b/dev-tools/cherrypick_pr index 24fec324a55..a148622c2a9 100755 --- a/dev-tools/cherrypick_pr +++ b/dev-tools/cherrypick_pr @@ -38,7 +38,7 @@ PR. def main(): """Main""" parser = argparse.ArgumentParser( - description="Creates a PR for merging two branches", + description="Creates a PR for cherry-picking commits", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=usage) parser.add_argument("to_branch", diff --git a/dev-tools/open_pr b/dev-tools/open_pr new file mode 100755 index 00000000000..b98599cab35 --- /dev/null +++ b/dev-tools/open_pr @@ -0,0 +1,102 @@ +#!/usr/bin/env python +"""Open a PR from the current branch""" + +import sys +import argparse +import requests +import re +from subprocess import check_call, check_output +from os.path import expanduser + +usage = """ +Example usage: + +Open PR against master, with the review label: + +./dev-tools/open_pr --remote tsg + +Open PR against the 6.0 branch, with the review and in progress labels: + +./dev-tools/open_pr --remote tsg --branch 6.0 --wip + +Open PR against the 6.0 branch, with the review and docs labels: + +./dev-tools/open_pr --remote tsg --labels=review,docs + +The title and message of the PR are taken from the _last_ commit in the +PR. +""" + +def main(): + parser = argparse.ArgumentParser( + description="Creates a PR against a given branch", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=usage) + + parser.add_argument("--remote", default="origin", + help="Your git remote to push the branch to.") + parser.add_argument("--branch", default="master", + help="Remote branch to open PR against (e.g 5.0)") + parser.add_argument("--wip", action="store_true", + help="Add the `in progress` label") + parser.add_argument("--labels", + help="Extra labels to add (comma separated)") + parser.add_argument("--yes", action="store_true", + help="Assume yes.") + args = parser.parse_args() + local_branch = check_output("git rev-parse --abbrev-ref HEAD", shell=True).strip() + title = check_output("git show -q HEAD --format='%s'", shell=True).strip() + msg = check_output("git show -q HEAD --format='%b'", shell=True).strip() + + labels = ["review"] + if args.labels: + labels.extend(args.labels.split(",")) + if args.wip: + lables += "in progress" + + print("Branch: {}".format(args.branch)) + print("Remote: {}".format(args.remote)) + print("Local branch: {}".format(local_branch)) + print("Title: {}".format(title)) + print("Message: {}".format(msg)) + print("Labels: {}".format(labels)) + + if not args.yes and raw_input("Continue? [Y/n]: ") not in ["y", "Y", ""]: + return 1 + + # push branch + check_call("git push --set-upstream {} {}" + .format(args.remote, local_branch), shell=True) + + + # open PR + token = open(expanduser("~/.elastic/github.token"), "r").read().strip() + base = "https://api.github.com/repos/elastic/beats" + session = requests.Session() + session.headers.update({"Authorization": "token " + token}) + + remote_url = check_output("git remote get-url {}".format(args.remote), + shell=True) + remote_user = re.search("github.com:(.+)/beats", remote_url).group(1) + + request = session.post(base + "/pulls", json=dict( + title=title, + head=remote_user + ":" + local_branch, + base=args.branch, + body=msg + )) + if request.status_code > 299: + print("Creating PR failed: {}".format(request.json())) + sys.exit(1) + new_pr = request.json() + + # add labels + session.post( + base + "/issues/{}/labels".format(new_pr["number"]), json=labels) + + print("\nDone. PR created: {}".format(new_pr["html_url"])) + print("Please go and review it for the message and labels.") + + +if __name__ == "__main__": + sys.exit(main()) From 6829fbc92133869b53ec17204bcb8ffb65a60d99 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Mon, 18 Sep 2017 14:18:50 +0200 Subject: [PATCH 26/96] Allow IndexPrefix to be set. (#5161) - Change defaults in reference.yml. - Set index-prefix to name by default. --- auditbeat/auditbeat.reference.yml | 2 +- filebeat/filebeat.reference.yml | 2 +- generator/metricbeat/{beat}/main.go.tmpl | 3 ++- heartbeat/heartbeat.reference.yml | 2 +- libbeat/_meta/config.reference.yml | 16 +++++++-------- libbeat/beat/info.go | 11 +++++----- libbeat/cmd/export.go | 6 +++--- libbeat/cmd/export/config.go | 4 ++-- libbeat/cmd/export/template.go | 6 +++--- libbeat/cmd/instance/beat.go | 20 +++++++++++-------- libbeat/cmd/instance/beat_test.go | 14 +++++++++++-- libbeat/cmd/modules.go | 2 +- libbeat/cmd/root.go | 10 +++++++--- libbeat/cmd/run.go | 4 ++-- libbeat/cmd/setup.go | 4 ++-- libbeat/cmd/test/config.go | 2 +- libbeat/cmd/test/output.go | 2 +- libbeat/cmd/version.go | 2 +- .../outputs/elasticsearch/elasticsearch.go | 2 +- libbeat/scripts/Makefile | 11 +++++----- libbeat/scripts/generate_index_pattern.py | 3 ++- libbeat/template/load.go | 2 +- libbeat/template/load_integration_test.go | 7 ++++--- metricbeat/cmd/test/modules.go | 2 +- metricbeat/metricbeat.reference.yml | 2 +- packetbeat/packetbeat.reference.yml | 2 +- winlogbeat/winlogbeat.reference.yml | 2 +- 27 files changed, 84 insertions(+), 61 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 6e37829e218..a44e8b11c2d 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -316,7 +316,7 @@ output.elasticsearch: # if no error is encountered. #slow_start: false - # Optional index name. The default index name is set to name of the beat + # Optional index name. The default index name is set to auditbeat # in all lowercase. #index: 'auditbeat' diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 05c76a7e7b6..95cbdcb530d 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -736,7 +736,7 @@ output.elasticsearch: # if no error is encountered. #slow_start: false - # Optional index name. The default index name is set to name of the beat + # Optional index name. The default index name is set to filebeat # in all lowercase. #index: 'filebeat' diff --git a/generator/metricbeat/{beat}/main.go.tmpl b/generator/metricbeat/{beat}/main.go.tmpl index f717f804273..4ebab803599 100644 --- a/generator/metricbeat/{beat}/main.go.tmpl +++ b/generator/metricbeat/{beat}/main.go.tmpl @@ -14,9 +14,10 @@ import ( ) var Name = "{beat}" +var IndexPrefix = "{beat}" func main() { - if err := instance.Run(Name, "", beater.New); err != nil { + if err := instance.Run(Name, IndexPrefix, "", beater.New); err != nil { os.Exit(1) } } diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 23350d2ac1c..73d3325b47f 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -465,7 +465,7 @@ output.elasticsearch: # if no error is encountered. #slow_start: false - # Optional index name. The default index name is set to name of the beat + # Optional index name. The default index name is set to heartbeat # in all lowercase. #index: 'heartbeat' diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 8248a2ead01..be10313e7a8 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -155,10 +155,10 @@ output.elasticsearch: # Number of workers per Elasticsearch host. #worker: 1 - # Optional index name. The default is "beatname" plus date - # and generates [beatname-]YYYY.MM.DD keys. + # Optional index name. The default is "beat-index-prefix" plus date + # and generates [beat-index-prefix-]YYYY.MM.DD keys. # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. - #index: "beatname-%{[beat.version]}-%{+yyyy.MM.dd}" + #index: "beat-index-prefix-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. #pipeline: "" @@ -251,9 +251,9 @@ output.elasticsearch: # if no error is encountered. #slow_start: false - # Optional index name. The default index name is set to name of the beat + # Optional index name. The default index name is set to beat-index-prefix # in all lowercase. - #index: 'beatname' + #index: 'beat-index-prefix' # SOCKS5 proxy server URL #proxy_url: socks5://user:password@socks5-server:2233 @@ -620,14 +620,14 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Template name. By default the template name is "beatname-%{[beat.version]}" +# Template name. By default the template name is "beat-index-prefix-%{[beat.version]}" # The template name and pattern has to be set in case the elasticsearch index pattern is modified. -#setup.template.name: "beatname-%{[beat.version]}" +#setup.template.name: "beat-index-prefix-%{[beat.version]}" # Template patttern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. # The first part is the version of the beat and then -* is used to match all daily indicies. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. -#setup.template.pattern: "beatname-%{[beat.version]}-*" +#setup.template.pattern: "beat-index-prefix-%{[beat.version]}-*" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" diff --git a/libbeat/beat/info.go b/libbeat/beat/info.go index 36c74e47f64..36fe041d1f4 100644 --- a/libbeat/beat/info.go +++ b/libbeat/beat/info.go @@ -4,9 +4,10 @@ import "github.com/satori/go.uuid" // Info stores a beats instance meta data. type Info struct { - Beat string // The actual beat's name - Version string // The beat version. Defaults to the libbeat version when an implementation does not set a version - Name string // configured beat name - Hostname string // hostname - UUID uuid.UUID // ID assigned to beat instance + Beat string // The actual beat's name + IndexPrefix string // The beat's index prefix in Elasticsearch. + Version string // The beat version. Defaults to the libbeat version when an implementation does not set a version + Name string // configured beat name + Hostname string // hostname + UUID uuid.UUID // ID assigned to beat instance } diff --git a/libbeat/cmd/export.go b/libbeat/cmd/export.go index 202d1c1f26c..6f33e8ad641 100644 --- a/libbeat/cmd/export.go +++ b/libbeat/cmd/export.go @@ -6,14 +6,14 @@ import ( "github.com/elastic/beats/libbeat/cmd/export" ) -func genExportCmd(name, beatVersion string) *cobra.Command { +func genExportCmd(name, idxPrefix, beatVersion string) *cobra.Command { exportCmd := &cobra.Command{ Use: "export", Short: "Export current config or index template", } - exportCmd.AddCommand(export.GenExportConfigCmd(name, beatVersion)) - exportCmd.AddCommand(export.GenTemplateConfigCmd(name, beatVersion)) + exportCmd.AddCommand(export.GenExportConfigCmd(name, idxPrefix, beatVersion)) + exportCmd.AddCommand(export.GenTemplateConfigCmd(name, idxPrefix, beatVersion)) return exportCmd } diff --git a/libbeat/cmd/export/config.go b/libbeat/cmd/export/config.go index f073455af9f..8687eef1c1a 100644 --- a/libbeat/cmd/export/config.go +++ b/libbeat/cmd/export/config.go @@ -10,12 +10,12 @@ import ( "github.com/elastic/beats/libbeat/cmd/instance" ) -func GenExportConfigCmd(name, beatVersion string) *cobra.Command { +func GenExportConfigCmd(name, idxPrefix, beatVersion string) *cobra.Command { return &cobra.Command{ Use: "config", Short: "Export current config to stdout", Run: func(cmd *cobra.Command, args []string) { - b, err := instance.NewBeat(name, beatVersion) + b, err := instance.NewBeat(name, idxPrefix, beatVersion) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go index e1f7c3ecf85..3fd10f1c533 100644 --- a/libbeat/cmd/export/template.go +++ b/libbeat/cmd/export/template.go @@ -11,7 +11,7 @@ import ( "github.com/elastic/beats/libbeat/template" ) -func GenTemplateConfigCmd(name, beatVersion string) *cobra.Command { +func GenTemplateConfigCmd(name, idxPrefix, beatVersion string) *cobra.Command { genTemplateConfigCmd := &cobra.Command{ Use: "template", Short: "Export index template to stdout", @@ -19,7 +19,7 @@ func GenTemplateConfigCmd(name, beatVersion string) *cobra.Command { version, _ := cmd.Flags().GetString("es.version") index, _ := cmd.Flags().GetString("index") - b, err := instance.NewBeat(name, beatVersion) + b, err := instance.NewBeat(name, idxPrefix, beatVersion) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) @@ -61,7 +61,7 @@ func GenTemplateConfigCmd(name, beatVersion string) *cobra.Command { } genTemplateConfigCmd.Flags().String("es.version", beatVersion, "Elasticsearch version") - genTemplateConfigCmd.Flags().String("index", name, "Base index name") + genTemplateConfigCmd.Flags().String("index", idxPrefix, "Base index name") return genTemplateConfigCmd } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 945e8b9bf6d..88fe86bff32 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -114,9 +114,9 @@ func initRand() { // implementation. bt is the `Creator` callback for creating a new beater // instance. // XXX Move this as a *Beat method? -func Run(name, version string, bt beat.Creator) error { +func Run(name, idxPrefix, version string, bt beat.Creator) error { return handleError(func() error { - b, err := NewBeat(name, version) + b, err := NewBeat(name, idxPrefix, version) if err != nil { return err } @@ -125,10 +125,13 @@ func Run(name, version string, bt beat.Creator) error { } // NewBeat creates a new beat instance -func NewBeat(name, v string) (*Beat, error) { +func NewBeat(name, indexPrefix, v string) (*Beat, error) { if v == "" { v = version.GetDefaultVersion() } + if indexPrefix == "" { + indexPrefix = name + } hostname, err := os.Hostname() if err != nil { @@ -137,11 +140,12 @@ func NewBeat(name, v string) (*Beat, error) { b := beat.Beat{ Info: beat.Info{ - Beat: name, - Version: v, - Name: hostname, - Hostname: hostname, - UUID: uuid.NewV4(), + Beat: name, + IndexPrefix: indexPrefix, + Version: v, + Name: hostname, + Hostname: hostname, + UUID: uuid.NewV4(), }, } diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go index a0c8eb96f41..412ed64f55f 100644 --- a/libbeat/cmd/instance/beat_test.go +++ b/libbeat/cmd/instance/beat_test.go @@ -10,21 +10,31 @@ import ( ) func TestNewInstance(t *testing.T) { - b, err := NewBeat("testbeat", "0.9") + b, err := NewBeat("testbeat", "testidx", "0.9") if err != nil { panic(err) } assert.Equal(t, "testbeat", b.Info.Beat) + assert.Equal(t, "testidx", b.Info.IndexPrefix) assert.Equal(t, "0.9", b.Info.Version) // UUID4 should be 36 chars long assert.Equal(t, 16, len(b.Info.UUID)) assert.Equal(t, 36, len(b.Info.UUID.String())) + + // indexPrefix set to name if empty + b, err = NewBeat("testbeat", "", "0.9") + if err != nil { + panic(err) + } + assert.Equal(t, "testbeat", b.Info.Beat) + assert.Equal(t, "testbeat", b.Info.IndexPrefix) + } func TestNewInstanceUUID(t *testing.T) { - b, err := NewBeat("testbeat", "0.9") + b, err := NewBeat("testbeat", "", "0.9") if err != nil { panic(err) } diff --git a/libbeat/cmd/modules.go b/libbeat/cmd/modules.go index 3ace488533b..cb896d06904 100644 --- a/libbeat/cmd/modules.go +++ b/libbeat/cmd/modules.go @@ -42,7 +42,7 @@ func GenModulesCmd(name, version string, modulesFactory modulesManagerFactory) * // Instantiate a modules manager or die trying func getModules(name, version string, modulesFactory modulesManagerFactory) ModulesManager { - b, err := instance.NewBeat(name, version) + b, err := instance.NewBeat(name, "", version) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/root.go b/libbeat/cmd/root.go index 4d23c582c97..31d8169c9b7 100644 --- a/libbeat/cmd/root.go +++ b/libbeat/cmd/root.go @@ -44,14 +44,18 @@ func GenRootCmd(name, version string, beatCreator beat.Creator) *BeatsRootCmd { // given (for backwards compatibility). runFlags parameter must the flagset used by // run command func GenRootCmdWithRunFlags(name, version string, beatCreator beat.Creator, runFlags *pflag.FlagSet) *BeatsRootCmd { + return GenRootCmdWithIndexPrefixWithRunFlags(name, name, version, beatCreator, runFlags) +} + +func GenRootCmdWithIndexPrefixWithRunFlags(name, indexPrefix, version string, beatCreator beat.Creator, runFlags *pflag.FlagSet) *BeatsRootCmd { rootCmd := &BeatsRootCmd{} rootCmd.Use = name - rootCmd.RunCmd = genRunCmd(name, version, beatCreator, runFlags) - rootCmd.SetupCmd = genSetupCmd(name, version, beatCreator) + rootCmd.RunCmd = genRunCmd(name, indexPrefix, version, beatCreator, runFlags) + rootCmd.SetupCmd = genSetupCmd(name, indexPrefix, version, beatCreator) rootCmd.VersionCmd = genVersionCmd(name, version) rootCmd.CompletionCmd = genCompletionCmd(name, version, rootCmd) - rootCmd.ExportCmd = genExportCmd(name, version) + rootCmd.ExportCmd = genExportCmd(name, indexPrefix, version) rootCmd.TestCmd = genTestCmd(name, version, beatCreator) // Root command is an alias for run diff --git a/libbeat/cmd/run.go b/libbeat/cmd/run.go index 4ba2ecb742a..9abe23f535a 100644 --- a/libbeat/cmd/run.go +++ b/libbeat/cmd/run.go @@ -11,12 +11,12 @@ import ( "github.com/elastic/beats/libbeat/cmd/instance" ) -func genRunCmd(name, version string, beatCreator beat.Creator, runFlags *pflag.FlagSet) *cobra.Command { +func genRunCmd(name, idxPrefix, version string, beatCreator beat.Creator, runFlags *pflag.FlagSet) *cobra.Command { runCmd := cobra.Command{ Use: "run", Short: "Run " + name, Run: func(cmd *cobra.Command, args []string) { - err := instance.Run(name, version, beatCreator) + err := instance.Run(name, idxPrefix, version, beatCreator) if err != nil { os.Exit(1) } diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index ee13cdba91c..b1b0306950a 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -10,7 +10,7 @@ import ( "github.com/elastic/beats/libbeat/cmd/instance" ) -func genSetupCmd(name, version string, beatCreator beat.Creator) *cobra.Command { +func genSetupCmd(name, idxPrefix, version string, beatCreator beat.Creator) *cobra.Command { setup := cobra.Command{ Use: "setup", Short: "Setup index template, dashboards and ML jobs", @@ -21,7 +21,7 @@ func genSetupCmd(name, version string, beatCreator beat.Creator) *cobra.Command * ML jobs (where available). `, Run: func(cmd *cobra.Command, args []string) { - beat, err := instance.NewBeat(name, version) + beat, err := instance.NewBeat(name, idxPrefix, version) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/test/config.go b/libbeat/cmd/test/config.go index c8cbc9cdb55..43e55b2e005 100644 --- a/libbeat/cmd/test/config.go +++ b/libbeat/cmd/test/config.go @@ -15,7 +15,7 @@ func GenTestConfigCmd(name, version string, beatCreator beat.Creator) *cobra.Com Use: "config", Short: "Test configuration settings", Run: func(cmd *cobra.Command, args []string) { - b, err := instance.NewBeat(name, version) + b, err := instance.NewBeat(name, "", version) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/test/output.go b/libbeat/cmd/test/output.go index cba7cb57db8..912509c7b6f 100644 --- a/libbeat/cmd/test/output.go +++ b/libbeat/cmd/test/output.go @@ -16,7 +16,7 @@ func GenTestOutputCmd(name, beatVersion string) *cobra.Command { Use: "output", Short: "Test " + name + " can connect to the output by using the current settings", Run: func(cmd *cobra.Command, args []string) { - b, err := instance.NewBeat(name, beatVersion) + b, err := instance.NewBeat(name, "", beatVersion) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/version.go b/libbeat/cmd/version.go index b34f37cad36..272e36fb85b 100644 --- a/libbeat/cmd/version.go +++ b/libbeat/cmd/version.go @@ -16,7 +16,7 @@ func genVersionCmd(name, beatVersion string) *cobra.Command { Use: "version", Short: "Show current version info", Run: func(cmd *cobra.Command, args []string) { - beat, err := instance.NewBeat(name, beatVersion) + beat, err := instance.NewBeat(name, "", beatVersion) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index bfc8dfc0650..720834b04bb 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -57,7 +57,7 @@ func makeES( } if !cfg.HasField("index") { - pattern := fmt.Sprintf("%v-%v-%%{+yyyy.MM.dd}", beat.Beat, beat.Version) + pattern := fmt.Sprintf("%v-%v-%%{+yyyy.MM.dd}", beat.IndexPrefix, beat.Version) cfg.SetString("index", -1, pattern) } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index db98d1088fd..6978ace061d 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -5,6 +5,7 @@ BEAT_TITLE?=${BEAT_NAME}## @packaging Title of the application BEAT_DESCRIPTION?=Sends events to Elasticsearch or Logstash ## @packaging Description of the application BEAT_PATH?=github.com/elastic/beats/${BEAT_NAME} BEAT_PACKAGE_NAME?=${BEAT_NAME} +BEAT_INDEX_PREFIX?=${BEAT_NAME} BEAT_URL?=https://www.elastic.co/products/beats/${BEAT_NAME} ## @packaging Link to the homepage of the application BEAT_DOC_URL?=https://www.elastic.co/guide/en/beats/${BEAT_NAME}/current/index.html ## @packaging Link to the user documentation of the application BEAT_LICENSE?=ASL 2.0 ## @packaging Software license of the application @@ -265,11 +266,11 @@ update: python-env collect @echo "Updating generated files for ${BEAT_NAME}" @# Update config files. - @cat _meta/beat.yml ${ES_BEATS}/libbeat/_meta/config.yml | sed -e "s/beatname/${BEAT_NAME}/g" > ${BEAT_NAME}.yml + @cat _meta/beat.yml ${ES_BEATS}/libbeat/_meta/config.yml | sed -e "s/beatname/${BEAT_NAME}/g;s/beat-index-prefix/${BEAT_INDEX_PREFIX}/g" > ${BEAT_NAME}.yml @if [ -e _meta/beat.reference.yml ]; then \ - cat _meta/beat.reference.yml ${ES_BEATS}/libbeat/_meta/config.reference.yml | sed -e "s/beatname/${BEAT_NAME}/g" > ${BEAT_NAME}.reference.yml ; \ + cat _meta/beat.reference.yml ${ES_BEATS}/libbeat/_meta/config.reference.yml | sed -e "s/beatname/${BEAT_NAME}/g;s/beat-index-prefix/${BEAT_INDEX_PREFIX}/g" > ${BEAT_NAME}.reference.yml ; \ else \ - cat _meta/beat.yml ${ES_BEATS}/libbeat/_meta/config.reference.yml | sed -e "s/beatname/${BEAT_NAME}/g" > ${BEAT_NAME}.reference.yml ; \ + cat _meta/beat.yml ${ES_BEATS}/libbeat/_meta/config.reference.yml | sed -e "s/beatname/${BEAT_NAME}/g;s/beat-index-prefix/${BEAT_INDEX_PREFIX}/g" > ${BEAT_NAME}.reference.yml ; \ fi; @chmod 0640 ${BEAT_NAME}.yml ${BEAT_NAME}.reference.yml @@ -291,7 +292,7 @@ endif @# Generate Kibana index pattern @mkdir -p $(PWD)/_meta/kibana/5.x/index-pattern @mkdir -p $(PWD)/_meta/kibana/default/index-pattern - @${PYTHON_ENV}/bin/python ${ES_BEATS}/libbeat/scripts/generate_index_pattern.py --index '${BEAT_NAME}-*' --beat $(PWD) --version ${BEAT_VERSION} + @${PYTHON_ENV}/bin/python ${ES_BEATS}/libbeat/scripts/generate_index_pattern.py --index '${BEAT_INDEX_PREFIX}-*' --beat-name ${BEAT_NAME} --beat $(PWD) --version ${BEAT_VERSION} # Runs the post-update target if it exists. Will not return errors @-$(MAKE) post-update @@ -308,7 +309,7 @@ docs-preview: ## @build Preview the documents for the beat in the browser .PHONY: index-template index-template: ## @build Generate index templates for the given $VERSION. This is for manual testing. - go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es${VERSION}.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml -es.version=${VERSION} + go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -index ${BEAT_INDEX_PREFIX} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es${VERSION}.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml -es.version=${VERSION} ### KIBANA FILES HANDLING ### ES_URL?=http://localhost:9200 diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py index 7ad6c2be65b..890ba07af17 100644 --- a/libbeat/scripts/generate_index_pattern.py +++ b/libbeat/scripts/generate_index_pattern.py @@ -161,7 +161,7 @@ def get_index_pattern_name(index): def dump_index_pattern(args, version, output): - fileName = get_index_pattern_name(args.index) + fileName = get_index_pattern_name(args.beat_name) target_dir = os.path.join(args.beat, "_meta", "kibana", version, "index-pattern") target_file = os.path.join(target_dir, fileName + ".json") @@ -187,6 +187,7 @@ def dump_index_pattern(args, version, output): parser.add_argument("--version", help="Beat version") parser.add_argument("--index", help="The name of the index-pattern") parser.add_argument("--beat", help="Local Beat directory") + parser.add_argument("--beat-name", help="Name of the beat") args = parser.parse_args() diff --git a/libbeat/template/load.go b/libbeat/template/load.go index a225b26f697..d596cf68cca 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -43,7 +43,7 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info) (*Loader // template is written to index func (l *Loader) Load() error { - tmpl, err := New(l.beatInfo.Version, l.beatInfo.Beat, l.client.GetVersion(), l.config) + tmpl, err := New(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), l.config) if err != nil { return fmt.Errorf("error creating template instance: %v", err) } diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 9e21e30036f..e7c316fbf2d 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -219,10 +219,11 @@ func TestOverwrite(t *testing.T) { } beatInfo := beat.Info{ - Beat: "testbeat", - Version: version.GetDefaultVersion(), + Beat: "testbeat", + IndexPrefix: "testbeatidx", + Version: version.GetDefaultVersion(), } - templateName := "testbeat-" + version.GetDefaultVersion() + templateName := "testbeatidx-" + version.GetDefaultVersion() absPath, err := filepath.Abs("../") assert.NotNil(t, absPath) diff --git a/metricbeat/cmd/test/modules.go b/metricbeat/cmd/test/modules.go index c977be61fb3..e6d45c80153 100644 --- a/metricbeat/cmd/test/modules.go +++ b/metricbeat/cmd/test/modules.go @@ -25,7 +25,7 @@ func GenTestModulesCmd(name, beatVersion string) *cobra.Command { filter_metricset = args[1] } - b, err := instance.NewBeat(name, beatVersion) + b, err := instance.NewBeat(name, "", beatVersion) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 6356dd3d70a..f8d138f6f1a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -695,7 +695,7 @@ output.elasticsearch: # if no error is encountered. #slow_start: false - # Optional index name. The default index name is set to name of the beat + # Optional index name. The default index name is set to metricbeat # in all lowercase. #index: 'metricbeat' diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 262cbce5c19..b2efb313313 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -703,7 +703,7 @@ output.elasticsearch: # if no error is encountered. #slow_start: false - # Optional index name. The default index name is set to name of the beat + # Optional index name. The default index name is set to packetbeat # in all lowercase. #index: 'packetbeat' diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 96ba9830c88..949abde3d9f 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -280,7 +280,7 @@ output.elasticsearch: # if no error is encountered. #slow_start: false - # Optional index name. The default index name is set to name of the beat + # Optional index name. The default index name is set to winlogbeat # in all lowercase. #index: 'winlogbeat' From 1f07cdce5fbdbec7a46c36d33b55a10ed8867f08 Mon Sep 17 00:00:00 2001 From: lcawley Date: Mon, 18 Sep 2017 15:51:22 -0700 Subject: [PATCH 27/96] [DOCS] Update link to certgen command --- libbeat/docs/shared-ssl-logstash-config.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libbeat/docs/shared-ssl-logstash-config.asciidoc b/libbeat/docs/shared-ssl-logstash-config.asciidoc index 8e55dc0c15d..b38a4f2f73e 100644 --- a/libbeat/docs/shared-ssl-logstash-config.asciidoc +++ b/libbeat/docs/shared-ssl-logstash-config.asciidoc @@ -20,7 +20,7 @@ To use SSL mutual authentication: document. There are many online resources available that describe how to create certificates. + TIP: If you are using X-Pack, you can use the -{securitydoc}/ssl-tls.html#generating-signed-certificates[certgen tool] to generate certificates. +{elasticsearch}/certgen.html[certgen tool] to generate certificates. . Configure {beatname_uc} to use SSL. In the +{beatname_lc}.yml+ config file, specify the following settings under `ssl`: @@ -138,4 +138,3 @@ the foreground so you can quickly see any errors that occur: Any errors will be printed to the console. See the <> for info about resolving common errors. - From bc98e1a95bd0f5fe8768454ee3bdac569e362c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 19 Sep 2017 16:45:02 +0200 Subject: [PATCH 28/96] Remove kibana `depends_on` elasticsearch (#5199) --- metricbeat/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 884930ee8f4..645ebafd35d 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -73,8 +73,6 @@ services: kibana: build: ./module/kibana/_meta - depends_on: - - elasticsearch kubernetes: build: ./module/kubernetes/_meta From 291cd9b50f35d4d2ddf0a4d9ab4ab975adec4246 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 20 Sep 2017 08:05:42 +0200 Subject: [PATCH 29/96] Ensure docs are build locally just as they are in prod (#5201) --- script/build_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build_docs.sh b/script/build_docs.sh index b5e29fbdece..7419a53dd08 100755 --- a/script/build_docs.sh +++ b/script/build_docs.sh @@ -25,7 +25,7 @@ echo "Index document: ${index}" dest_dir="$html_dir/${name}" mkdir -p "$dest_dir" -params="" +params="--chunk=1" if [ "$PREVIEW" = "1" ]; then params="--chunk=1 -open chunk=1 -open" fi From bc84970df6de1e4017cdd1718358f13c60cae92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 20 Sep 2017 08:06:35 +0200 Subject: [PATCH 30/96] Update `beta` messages for config reload (#5202) --- filebeat/crawler/crawler.go | 5 ----- libbeat/cfgfile/reload.go | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/filebeat/crawler/crawler.go b/filebeat/crawler/crawler.go index 891880a400a..4582ccedf67 100644 --- a/filebeat/crawler/crawler.go +++ b/filebeat/crawler/crawler.go @@ -11,7 +11,6 @@ import ( "github.com/elastic/beats/filebeat/registrar" "github.com/elastic/beats/libbeat/cfgfile" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" _ "github.com/elastic/beats/filebeat/include" @@ -55,8 +54,6 @@ func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config } if configProspectors.Enabled() { - cfgwarn.Beta("Loading separate prospectors is enabled.") - c.prospectorsReloader = cfgfile.NewReloader(configProspectors) runnerFactory := prospector.NewRunnerFactory(c.out, r, c.beatDone) if err := c.prospectorsReloader.Check(runnerFactory); err != nil { @@ -69,8 +66,6 @@ func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config } if configModules.Enabled() { - cfgwarn.Beta("Loading separate modules is enabled.") - c.modulesReloader = cfgfile.NewReloader(configModules) modulesFactory := fileset.NewFactory(c.out, r, c.beatVersion, pipelineLoaderFactory, c.beatDone) if err := c.modulesReloader.Check(modulesFactory); err != nil { diff --git a/libbeat/cfgfile/reload.go b/libbeat/cfgfile/reload.go index 916b5e70f41..5560b2ca527 100644 --- a/libbeat/cfgfile/reload.go +++ b/libbeat/cfgfile/reload.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" "github.com/elastic/beats/libbeat/paths" @@ -73,6 +74,10 @@ func NewReloader(cfg *common.Config) *Reloader { path = paths.Resolve(paths.Config, path) } + if config.Reload.Enabled { + cfgwarn.Beta("Dynamic config reload is enabled.") + } + return &Reloader{ registry: NewRegistry(), config: config, From c0de0626d479d68f5d663078d5597248b6235707 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Wed, 20 Sep 2017 08:08:05 +0200 Subject: [PATCH 31/96] Fix packetbeat flaky tests (#5200) Moved the shutdown timeout after stopping the flows, but before stopping the transaction publishing. It needs to be after stopping the flows because the tests rely on the shutdown sequence to create flow reports. --- packetbeat/beater/packetbeat.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go index 5dfee9c017c..768ca7e790a 100644 --- a/packetbeat/beater/packetbeat.go +++ b/packetbeat/beater/packetbeat.go @@ -183,6 +183,12 @@ func (pb *packetbeat) Run(b *beat.Beat) error { } defer pb.transPub.Stop() + + timeout := pb.config.ShutdownTimeout + if timeout > 0 { + defer time.Sleep(timeout) + } + if pb.flows != nil { pb.flows.Start() defer pb.flows.Stop() @@ -210,11 +216,6 @@ func (pb *packetbeat) Run(b *beat.Beat) error { return err } - timeout := pb.config.ShutdownTimeout - if timeout > 0 { - time.Sleep(timeout) - } - return nil } From 23f6efeb2ac1749b7abd77ddadc30ffbf6b649d6 Mon Sep 17 00:00:00 2001 From: Monica Sarbu Date: Wed, 20 Sep 2017 08:10:04 +0200 Subject: [PATCH 32/96] Fix default paths in macOS for Redis 4.0.1 (#5173) --- CHANGELOG.asciidoc | 1 + filebeat/docs/modules/redis.asciidoc | 5 +++-- filebeat/filebeat.reference.yml | 2 +- filebeat/module/redis/_meta/config.reference.yml | 2 +- filebeat/module/redis/_meta/config.yml | 2 +- filebeat/module/redis/_meta/docs.asciidoc | 5 +++-- filebeat/module/redis/log/manifest.yml | 1 + filebeat/modules.d/redis.yml.disabled | 2 +- 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 4405dc482f8..44f533a9270 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -43,6 +43,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di *Auditbeat* *Filebeat* +- Fix default paths for redis 4.0.1 logs on macOS {pull}5173[5173] *Heartbeat* diff --git a/filebeat/docs/modules/redis.asciidoc b/filebeat/docs/modules/redis.asciidoc index b1c10113628..7634db1ac64 100644 --- a/filebeat/docs/modules/redis.asciidoc +++ b/filebeat/docs/modules/redis.asciidoc @@ -13,8 +13,9 @@ This module has two filesets: * The `slowlog` fileset connects to Redis via the network and retrieves the slow logs by using the `SLOWLOG` command. -For the `log` fileset, make sure the `logfile` option is set in the Redis configuration file. For -the `slowlog` fileset, make sure the `slowlog-log-slower-than` is set. +For the `log` fileset, make sure the `logfile` option, from the Redis configuration file, is set to `redis-server.log`. + +For the `slowlog` fileset, make sure the `slowlog-log-slower-than` option, from the Redis configuration file, is set to a lower value than the default one. [float] === Compatibility diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 95cbdcb530d..6c9f03a3ace 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -210,7 +210,7 @@ filebeat.modules: #enabled: true # The Redis hosts to connect to. - #var.hosts: ["localhost:6378"] + #var.hosts: ["localhost:6379"] # Optional, the password to use when connecting to Redis. #var.password: diff --git a/filebeat/module/redis/_meta/config.reference.yml b/filebeat/module/redis/_meta/config.reference.yml index 1bf9495f67a..cf8acdb62a9 100644 --- a/filebeat/module/redis/_meta/config.reference.yml +++ b/filebeat/module/redis/_meta/config.reference.yml @@ -12,7 +12,7 @@ #enabled: true # The Redis hosts to connect to. - #var.hosts: ["localhost:6378"] + #var.hosts: ["localhost:6379"] # Optional, the password to use when connecting to Redis. #var.password: diff --git a/filebeat/module/redis/_meta/config.yml b/filebeat/module/redis/_meta/config.yml index cc1912a44de..4aa2f1eacf0 100644 --- a/filebeat/module/redis/_meta/config.yml +++ b/filebeat/module/redis/_meta/config.yml @@ -12,7 +12,7 @@ enabled: true # The Redis hosts to connect to. - #var.hosts: ["localhost:6378"] + #var.hosts: ["localhost:6379"] # Optional, the password to use when connecting to Redis. #var.password: diff --git a/filebeat/module/redis/_meta/docs.asciidoc b/filebeat/module/redis/_meta/docs.asciidoc index fda925d0ff6..6bb3d613675 100644 --- a/filebeat/module/redis/_meta/docs.asciidoc +++ b/filebeat/module/redis/_meta/docs.asciidoc @@ -8,8 +8,9 @@ This module has two filesets: * The `slowlog` fileset connects to Redis via the network and retrieves the slow logs by using the `SLOWLOG` command. -For the `log` fileset, make sure the `logfile` option is set in the Redis configuration file. For -the `slowlog` fileset, make sure the `slowlog-log-slower-than` is set. +For the `log` fileset, make sure the `logfile` option, from the Redis configuration file, is set to `redis-server.log`. + +For the `slowlog` fileset, make sure the `slowlog-log-slower-than` option, from the Redis configuration file, is set to a lower value than the default one. [float] === Compatibility diff --git a/filebeat/module/redis/log/manifest.yml b/filebeat/module/redis/log/manifest.yml index 4aed1d84d4e..98351f14549 100644 --- a/filebeat/module/redis/log/manifest.yml +++ b/filebeat/module/redis/log/manifest.yml @@ -6,6 +6,7 @@ var: - /var/log/redis/redis-server.log* os.darwin: - /usr/local/var/log/redis/redis-server.log* + - /usr/local/var/db/redis/redis-server.log* os.windows: - "c:/program files/Redis/logs/redis.log*" diff --git a/filebeat/modules.d/redis.yml.disabled b/filebeat/modules.d/redis.yml.disabled index cc1912a44de..4aa2f1eacf0 100644 --- a/filebeat/modules.d/redis.yml.disabled +++ b/filebeat/modules.d/redis.yml.disabled @@ -12,7 +12,7 @@ enabled: true # The Redis hosts to connect to. - #var.hosts: ["localhost:6378"] + #var.hosts: ["localhost:6379"] # Optional, the password to use when connecting to Redis. #var.password: From c2bd3efb2994606731b25e1a761b877f3b70dd25 Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Wed, 20 Sep 2017 02:19:38 -0700 Subject: [PATCH 33/96] Remove ID from Runner interface (#5158) --- CHANGELOG.asciidoc | 1 + filebeat/crawler/crawler.go | 6 +++--- filebeat/prospector/prospector.go | 13 ++++--------- libbeat/cfgfile/reload.go | 1 - metricbeat/mb/module/runner.go | 7 ------- metricbeat/mb/module/wrapper.go | 23 ----------------------- 6 files changed, 8 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 44f533a9270..cf55fc9daff 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -39,6 +39,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Keep Docker & Kubernetes pod metadata after container dies while they are needed by processors. {pull}5084[5084] - Fix `fields.yml` lookup when using `export template` with a custom `path.config` param. {issue}5089[5089] - Remove runner creation from every reload check {pull}5141[5141] +- Remove ID() from Runner interface {issue}5153[5153] *Auditbeat* diff --git a/filebeat/crawler/crawler.go b/filebeat/crawler/crawler.go index 4582ccedf67..4adac5a9216 100644 --- a/filebeat/crawler/crawler.go +++ b/filebeat/crawler/crawler.go @@ -92,11 +92,11 @@ func (c *Crawler) startProspector(config *common.Config, states []file.State) er } p.Once = c.once - if _, ok := c.prospectors[p.ID()]; ok { - return fmt.Errorf("Prospector with same ID already exists: %v", p.ID()) + if _, ok := c.prospectors[p.ID]; ok { + return fmt.Errorf("Prospector with same ID already exists: %d", p.ID) } - c.prospectors[p.ID()] = p + c.prospectors[p.ID] = p p.Start() diff --git a/filebeat/prospector/prospector.go b/filebeat/prospector/prospector.go index 5ed9fc77187..700d7e0c09d 100644 --- a/filebeat/prospector/prospector.go +++ b/filebeat/prospector/prospector.go @@ -25,7 +25,7 @@ type Prospector struct { prospectorer Prospectorer done chan struct{} wg *sync.WaitGroup - id uint64 + ID uint64 Once bool beatDone chan struct{} } @@ -52,7 +52,7 @@ func New( var h map[string]interface{} conf.Unpack(&h) - prospector.id, err = hashstructure.Hash(h, nil) + prospector.ID, err = hashstructure.Hash(h, nil) if err != nil { return nil, err } @@ -81,7 +81,7 @@ func New( // Start starts the prospector func (p *Prospector) Start() { p.wg.Add(1) - logp.Info("Starting prospector of type: %v; id: %v ", p.config.Type, p.ID()) + logp.Info("Starting prospector of type: %v; ID: %d ", p.config.Type, p.ID) onceWg := sync.WaitGroup{} if p.Once { @@ -124,11 +124,6 @@ func (p *Prospector) Run() { } } -// ID returns prospector identifier -func (p *Prospector) ID() uint64 { - return p.id -} - // Stop stops the prospector and with it all harvesters func (p *Prospector) Stop() { // Stop scanning and wait for completion @@ -137,7 +132,7 @@ func (p *Prospector) Stop() { } func (p *Prospector) stop() { - logp.Info("Stopping Prospector: %v", p.ID()) + logp.Info("Stopping Prospector: %d", p.ID) // In case of once, it will be waited until harvesters close itself if p.Once { diff --git a/libbeat/cfgfile/reload.go b/libbeat/cfgfile/reload.go index 5560b2ca527..b2f0e5ec73f 100644 --- a/libbeat/cfgfile/reload.go +++ b/libbeat/cfgfile/reload.go @@ -52,7 +52,6 @@ type RunnerFactory interface { type Runner interface { Start() Stop() - ID() uint64 } // Reloader is used to register and reload modules diff --git a/metricbeat/mb/module/runner.go b/metricbeat/mb/module/runner.go index 90f6763b73a..a09ebf59597 100644 --- a/metricbeat/mb/module/runner.go +++ b/metricbeat/mb/module/runner.go @@ -17,9 +17,6 @@ type Runner interface { // publisher.Client will be closed by Stop. If Stop is called more than // once, only the first stop the Module and wait for it to exit. Stop() - - // Added to be consistent with cfgfile.Runner - ID() uint64 } // NewRunner returns a Runner facade. The events generated by @@ -60,7 +57,3 @@ func (mr *runner) Stop() { mr.wg.Wait() }) } - -func (mr *runner) ID() uint64 { - return mr.mod.Hash() -} diff --git a/metricbeat/mb/module/wrapper.go b/metricbeat/mb/module/wrapper.go index a2399d2fcaa..1c6ad5c12ed 100644 --- a/metricbeat/mb/module/wrapper.go +++ b/metricbeat/mb/module/wrapper.go @@ -12,8 +12,6 @@ import ( "github.com/elastic/beats/libbeat/monitoring" "github.com/elastic/beats/libbeat/testing" "github.com/elastic/beats/metricbeat/mb" - - "github.com/mitchellh/hashstructure" ) // Expvar metric names. @@ -37,7 +35,6 @@ var ( type Wrapper struct { mb.Module metricSets []*metricSetWrapper // List of pointers to its associated MetricSets. - configHash uint64 maxStartDelay time.Duration } @@ -127,26 +124,6 @@ func (mw *Wrapper) String() string { mw.Name(), len(mw.metricSets)) } -// Hash returns the hash value of the module wrapper -// This allows to check if two modules are the same / have the same config -func (mw *Wrapper) Hash() uint64 { - // Check if hash was calculated previously - if mw.configHash > 0 { - return mw.configHash - } - var err error - - // Config is unpacked into map[string]interface{} to also take metricset - // configs into account for the hash. - var c map[string]interface{} - mw.UnpackConfig(&c) - mw.configHash, err = hashstructure.Hash(c, nil) - if err != nil { - logp.Err("Error creating config hash for module %s: %s", mw.String(), err) - } - return mw.configHash -} - // MetricSets return the list of metricsets of the module func (mw *Wrapper) MetricSets() []*metricSetWrapper { return mw.metricSets From 25b90ae8f411558575bd746e64c288e8a7beb3e1 Mon Sep 17 00:00:00 2001 From: kel Date: Wed, 20 Sep 2017 22:12:39 +0800 Subject: [PATCH 34/96] [Logging] Include Logstash host info in debug messages #5198 (#5208) --- libbeat/outputs/logstash/async.go | 8 ++++---- libbeat/outputs/logstash/sync.go | 10 +++++----- libbeat/outputs/transport/client.go | 4 ++++ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/libbeat/outputs/logstash/async.go b/libbeat/outputs/logstash/async.go index 70eb6609f58..21e96c42927 100644 --- a/libbeat/outputs/logstash/async.go +++ b/libbeat/outputs/logstash/async.go @@ -139,8 +139,8 @@ func (c *asyncClient) Publish(batch publisher.Batch) error { n, err = c.publishWindowed(ref, events) } - debugf("%v events out of %v events sent to logstash. Continue sending", - n, len(events)) + debugf("%v events out of %v events sent to logstash host %s. Continue sending", + n, len(events), c.Host()) events = events[n:] if err != nil { @@ -159,8 +159,8 @@ func (c *asyncClient) publishWindowed( batchSize := len(events) windowSize := c.win.get() - debugf("Try to publish %v events to logstash with window size %v", - batchSize, windowSize) + debugf("Try to publish %v events to logstash host %s with window size %v", + batchSize, c.Host(), windowSize) // prepare message payload if batchSize > windowSize { diff --git a/libbeat/outputs/logstash/sync.go b/libbeat/outputs/logstash/sync.go index 08bf181b776..d1e9e590757 100644 --- a/libbeat/outputs/logstash/sync.go +++ b/libbeat/outputs/logstash/sync.go @@ -76,7 +76,7 @@ func (c *syncClient) Close() error { func (c *syncClient) reconnect() error { if err := c.Client.Close(); err != nil { - logp.Err("error closing connection to logstash: %s, reconnecting...", err) + logp.Err("error closing connection to logstash host %s: %s, reconnecting...", c.Host(), err) } return c.Client.Connect() } @@ -123,8 +123,8 @@ func (c *syncClient) Publish(batch publisher.Batch) error { events = events[n:] st.Acked(n) - debugf("%v events out of %v events sent to logstash. Continue sending", - n, len(events)) + debugf("%v events out of %v events sent to logstash host %s. Continue sending", + n, len(events), c.Host()) if err != nil { // return batch to pipeline before reporting/counting error @@ -151,8 +151,8 @@ func (c *syncClient) Publish(batch publisher.Batch) error { func (c *syncClient) publishWindowed(events []publisher.Event) (int, error) { batchSize := len(events) windowSize := c.win.get() - debugf("Try to publish %v events to logstash with window size %v", - batchSize, windowSize) + debugf("Try to publish %v events to logstash host %s with window size %v", + batchSize, c.Host(), windowSize) // prepare message payload if batchSize > windowSize { diff --git a/libbeat/outputs/transport/client.go b/libbeat/outputs/transport/client.go index 421fbd1c764..2b3b6f1d3c0 100644 --- a/libbeat/outputs/transport/client.go +++ b/libbeat/outputs/transport/client.go @@ -162,6 +162,10 @@ func (c *Client) RemoteAddr() net.Addr { return nil } +func (c *Client) Host() string { + return c.host +} + func (c *Client) SetDeadline(t time.Time) error { conn := c.getConn() if conn == nil { From 3effafdf5c4a2138031e5906be471acc99f75df0 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 21 Sep 2017 10:41:19 +0200 Subject: [PATCH 35/96] Add LICENSE.txt into the packages (#5215) We had it in the repository, but wasn't deployed in the packages. This corrects this. It also renamed LICENSE and NOTICE to LICENSE.txt/NOTICE.txt to align with Elasticsearch and Kibana. --- LICENSE => LICENSE.txt | 2 +- NOTICE => NOTICE.txt | 0 dev-tools/generate_notice.py | 2 +- libbeat/scripts/Makefile | 6 +++++- 4 files changed, 7 insertions(+), 3 deletions(-) rename LICENSE => LICENSE.txt (89%) rename NOTICE => NOTICE.txt (100%) diff --git a/LICENSE b/LICENSE.txt similarity index 89% rename from LICENSE rename to LICENSE.txt index 43976b73b2b..edfa6086a81 100644 --- a/LICENSE +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2012–2016 Elasticsearch +Copyright (c) 2012–2017 Elastic Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/NOTICE b/NOTICE.txt similarity index 100% rename from NOTICE rename to NOTICE.txt diff --git a/dev-tools/generate_notice.py b/dev-tools/generate_notice.py index 3fdbf34a622..7c3966612ce 100644 --- a/dev-tools/generate_notice.py +++ b/dev-tools/generate_notice.py @@ -231,7 +231,7 @@ def detect_license_summary(content): args = parser.parse_args() cwd = os.getcwd() - notice = os.path.join(cwd, "NOTICE") + notice = os.path.join(cwd, "NOTICE.txt") vendor_dirs = [] excludes = args.excludes diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 6978ace061d..1591d5f4f4c 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -14,7 +14,8 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) ES_BEATS?=..## @community_beat Must be set to ./vendor/github.com/elastic/beats. It must always be a relative path. GOPACKAGES?=$(shell go list ${BEAT_PATH}/... | grep -v /vendor/) PACKER_TEMPLATES_DIR?=${ES_BEATS}/dev-tools/packer ## @Building Directory of templates that are used by "make package" -NOTICE_FILE?=../NOTICE +NOTICE_FILE?=../NOTICE.txt +LICENSE_FILE?=../LICENSE.txt space:=$() # comma:=, @@ -363,6 +364,9 @@ install-home: if [ -a ${NOTICE_FILE} ]; then \ install -m 644 ${NOTICE_FILE} ${HOME_PREFIX}/; \ fi + if [ -a ${LICENSE_FILE} ]; then \ + install -m 644 ${LICENSE_FILE} ${HOME_PREFIX}/; \ + fi if [ -d _meta/module.generated ]; then \ install -d -m 755 ${HOME_PREFIX}/module; \ rsync -av _meta/module.generated/ ${HOME_PREFIX}/module/; \ From b0757d250d52509deaaa03c60667ac3f95fe9675 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 21 Sep 2017 10:43:18 +0200 Subject: [PATCH 36/96] Do not clean ignored files in merge_pr script (#5195) The merge_pr script did also remove all git ignored files. This lead to issues with the IDE and some of my test files like `filebeat.dev.yml` were also removed. We removed this in the past from cherrypick_pr so I assume its ok to also remove it here. --- dev-tools/merge_pr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/merge_pr b/dev-tools/merge_pr index 3f6b0f3bd60..2d6f119ccb2 100755 --- a/dev-tools/merge_pr +++ b/dev-tools/merge_pr @@ -25,7 +25,7 @@ def main(): "Continue? [y/n]: ") != "y": return 1 check_call("git reset --hard", shell=True) - check_call("git clean -dfx", shell=True) + check_call("git clean -df", shell=True) check_call("git fetch", shell=True) check_call("git checkout {}".format(args.from_branch), shell=True) From 10af6f9c03b3ef50759bc3bbc80c19b9d51ba9d0 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Thu, 21 Sep 2017 03:42:58 -0700 Subject: [PATCH 37/96] Remove changeme password from the docs (#5214) --- auditbeat/docs/index.asciidoc | 2 ++ filebeat/docs/index.asciidoc | 2 ++ heartbeat/docs/index.asciidoc | 2 ++ libbeat/docs/breaking.asciidoc | 4 ++-- libbeat/docs/index.asciidoc | 2 ++ libbeat/docs/outputconfig.asciidoc | 10 +++++++--- metricbeat/docs/index.asciidoc | 2 ++ packetbeat/docs/index.asciidoc | 2 ++ winlogbeat/docs/index.asciidoc | 2 ++ 9 files changed, 23 insertions(+), 5 deletions(-) diff --git a/auditbeat/docs/index.asciidoc b/auditbeat/docs/index.asciidoc index 5b52054e561..1017437707e 100644 --- a/auditbeat/docs/index.asciidoc +++ b/auditbeat/docs/index.asciidoc @@ -2,6 +2,8 @@ include::../../libbeat/docs/version.asciidoc[] +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + :libbeat: http://www.elastic.co/guide/en/beats/libbeat/{doc-branch} :kibana-ref: https://www.elastic.co/guide/en/kibana/{doc-branch} :beatsdevguide: http://www.elastic.co/guide/en/beats/devguide/{doc-branch} diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index bac7ea45adb..43ac7587b02 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -2,6 +2,8 @@ include::../../libbeat/docs/version.asciidoc[] +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + :libbeat: http://www.elastic.co/guide/en/beats/libbeat/{doc-branch} :kibana-ref: https://www.elastic.co/guide/en/kibana/{doc-branch} :beatsdevguide: http://www.elastic.co/guide/en/beats/devguide/{doc-branch} diff --git a/heartbeat/docs/index.asciidoc b/heartbeat/docs/index.asciidoc index 300986c7fe2..819153a9c33 100644 --- a/heartbeat/docs/index.asciidoc +++ b/heartbeat/docs/index.asciidoc @@ -2,6 +2,8 @@ include::../../libbeat/docs/version.asciidoc[] +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + :libbeat: http://www.elastic.co/guide/en/beats/libbeat/{doc-branch} :kibana-ref: https://www.elastic.co/guide/en/kibana/{doc-branch} :beatsdevguide: http://www.elastic.co/guide/en/beats/devguide/{doc-branch} diff --git a/libbeat/docs/breaking.asciidoc b/libbeat/docs/breaking.asciidoc index b1727397cec..adbbcc97faa 100644 --- a/libbeat/docs/breaking.asciidoc +++ b/libbeat/docs/breaking.asciidoc @@ -119,14 +119,14 @@ following command: ["source","sh",subs="attributes"] ---- -./scripts/import_dashboards -user elastic -pass changeme +./scripts/import_dashboards -user elastic -pass {pwd} ---- Can be replaced with: ["source","sh",subs="attributes"] ---- -./filebeat setup -E "output.elasticsearch.username=elastic" -E "output.elasticsearch.password=changeme" +./filebeat setup -E "output.elasticsearch.username=elastic" -E "output.elasticsearch.password={pwd}" ---- Note that the `-E` flags are only required if the Elasticsearch output is not diff --git a/libbeat/docs/index.asciidoc b/libbeat/docs/index.asciidoc index 690b0fd2c7f..ffa7ec756f8 100644 --- a/libbeat/docs/index.asciidoc +++ b/libbeat/docs/index.asciidoc @@ -3,6 +3,8 @@ include::./version.asciidoc[] +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + :packetbeat: http://www.elastic.co/guide/en/beats/packetbeat/{doc-branch} :metricbeat: http://www.elastic.co/guide/en/beats/metricbeat/{doc-branch} :filebeat: http://www.elastic.co/guide/en/beats/filebeat/{doc-branch} diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 3578f503f5b..b30d126ac09 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -1101,22 +1101,26 @@ output.console: [[configure-cloud-id]] === Configure the output for the Elastic Cloud +++++ +Cloud +++++ + {beatname_uc} comes with two settings that simplify the output configuration when used together with https://cloud.elastic.co/[Elastic Cloud]. When defined, these setting overwrite settings from other parts in the configuration. Example: -[source,yaml] +["source","yaml",subs="attributes"] ------------------------------------------------------------------------------ cloud.id: "staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==" -cloud.auth: "elastic:changeme" +cloud.auth: "elastic:{pwd}" ------------------------------------------------------------------------------ These settings can be also specified at the command line, like this: -["source","sh",subs="attributes,callouts"] +["source","sh",subs="attributes"] ------------------------------------------------------------------------------ {beatname_lc} -e -E cloud.id="" -E cloud.auth="" ------------------------------------------------------------------------------ diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 3dbd7407f99..2aa60b2e81f 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -2,6 +2,8 @@ include::../../libbeat/docs/version.asciidoc[] +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + :libbeat: http://www.elastic.co/guide/en/beats/libbeat/{doc-branch} :kibana-ref: https://www.elastic.co/guide/en/kibana/{doc-branch} :beatsdevguide: http://www.elastic.co/guide/en/beats/devguide/{doc-branch} diff --git a/packetbeat/docs/index.asciidoc b/packetbeat/docs/index.asciidoc index 7ef49bd6243..05a3e521307 100644 --- a/packetbeat/docs/index.asciidoc +++ b/packetbeat/docs/index.asciidoc @@ -2,6 +2,8 @@ include::../../libbeat/docs/version.asciidoc[] +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + :libbeat: http://www.elastic.co/guide/en/beats/libbeat/{doc-branch} :kibana-ref: https://www.elastic.co/guide/en/kibana/{doc-branch} :beatsdevguide: http://www.elastic.co/guide/en/beats/devguide/{doc-branch} diff --git a/winlogbeat/docs/index.asciidoc b/winlogbeat/docs/index.asciidoc index 4919f504179..fef85a09bd3 100644 --- a/winlogbeat/docs/index.asciidoc +++ b/winlogbeat/docs/index.asciidoc @@ -2,6 +2,8 @@ include::../../libbeat/docs/version.asciidoc[] +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + :libbeat: http://www.elastic.co/guide/en/beats/libbeat/{doc-branch} :kibana-ref: https://www.elastic.co/guide/en/kibana/{doc-branch} :beatsdevguide: http://www.elastic.co/guide/en/beats/devguide/{doc-branch} From 22e46edb7b9e6e5ecd44a4332fb2f326048910d8 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 21 Sep 2017 15:39:37 +0200 Subject: [PATCH 38/96] Fix flaky reload tests (#5221) Because of the new publisher, sometimes events were still written to disk after the reloading stopped message. This is fixed by setting the flush size to 1 so events are immidiately flushed. --- metricbeat/tests/system/config/metricbeat.yml.j2 | 2 +- metricbeat/tests/system/test_reload.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index 2c02f4b78c9..d3d5f6b09b5 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -143,7 +143,7 @@ processors: queue.mem: events: 4096 - flush.min_events: 8 + flush.min_events: {{ flush_min_events|default(8) }} flush.timeout: 0.1s #================================ Outputs ===================================== diff --git a/metricbeat/tests/system/test_reload.py b/metricbeat/tests/system/test_reload.py index 94637269e68..d5734b5a435 100644 --- a/metricbeat/tests/system/test_reload.py +++ b/metricbeat/tests/system/test_reload.py @@ -24,6 +24,7 @@ def test_reload(self): self.render_config_template( reload=True, reload_path=self.working_dir + "/configs/*.yml", + flush_min_events=1, ) proc = self.start_beat() @@ -49,6 +50,7 @@ def test_start_stop(self): self.render_config_template( reload=True, reload_path=self.working_dir + "/configs/*.yml", + flush_min_events=1, ) os.mkdir(self.working_dir + "/configs/") From 57aaa90929a49ba9b11cdc800c49930df3e24161 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Thu, 21 Sep 2017 15:59:54 +0200 Subject: [PATCH 39/96] Allow for more than one index file in docs folders. (#5220) --- script/build_docs.sh | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/script/build_docs.sh b/script/build_docs.sh index 7419a53dd08..b54cee2294d 100755 --- a/script/build_docs.sh +++ b/script/build_docs.sh @@ -17,17 +17,19 @@ else echo "$docs_dir already exists. Not cloning." fi - -index="${GOPATH%%:*}/src/${path}/index.asciidoc" - -echo "Building docs for ${name}..." -echo "Index document: ${index}" - -dest_dir="$html_dir/${name}" -mkdir -p "$dest_dir" -params="--chunk=1" -if [ "$PREVIEW" = "1" ]; then - params="--chunk=1 -open chunk=1 -open" -fi -$docs_dir/build_docs.pl $params --doc "$index" -out "$dest_dir" - +index_list="$(find ${GOPATH%%:*}/src/$path -name 'index.asciidoc')" +for index in $index_list +do + echo "Building docs for ${name}..." + echo "Index document: ${index}" + index_path=$(basename $(dirname $index)) + echo "Index path: $index_path" + + dest_dir="$html_dir/${name}/${index_path}" + mkdir -p "$dest_dir" + params="--chunk=1" + if [ "$PREVIEW" = "1" ]; then + params="--chunk=1 -open chunk=1 -open" + fi + $docs_dir/build_docs.pl $params --doc "$index" -out "$dest_dir" +done From ae2f13b2ed635a046a041221138166a8d955b1d3 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 25 Sep 2017 10:49:52 +0200 Subject: [PATCH 40/96] Added the 5.x version of the Auditbeat dashboards (#5222) Exported the current dashboards from master using Kibana 5.6 and stored them in the 5.x folder of the module. Also upgrades the 5x.yml testing env to use 5.6.1 --- .../kibana/5.x/dashboard/AV0tXkjYg1PYniApZbKP.json | 13 +++++++++++++ .../5.x/visualization/AV0tV05vg1PYniApZbA2.json | 10 ++++++++++ .../5.x/visualization/AV0tVcg6g1PYniApZa-v.json | 10 ++++++++++ .../5.x/visualization/AV0tW0djg1PYniApZbGL.json | 10 ++++++++++ .../5.x/visualization/AV0tWL-Yg1PYniApZbCs.json | 10 ++++++++++ .../5.x/visualization/AV0tWSdXg1PYniApZbDU.json | 10 ++++++++++ .../5.x/visualization/AV0tY6jwg1PYniApZbRY.json | 10 ++++++++++ .../5.x/visualization/AV0tav8Ag1PYniApZbbK.json | 10 ++++++++++ .../5.x/visualization/AV0tbcUdg1PYniApZbe1.json | 10 ++++++++++ .../5.x/visualization/AV0tc_xZg1PYniApZbnL.json | 10 ++++++++++ .../5.x/visualization/AV0te0TCg1PYniApZbw9.json | 10 ++++++++++ .../5.x/visualization/AV0tes4Eg1PYniApZbwV.json | 10 ++++++++++ testing/environments/5x.yml | 6 +++--- 13 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/dashboard/AV0tXkjYg1PYniApZbKP.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tV05vg1PYniApZbA2.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tVcg6g1PYniApZa-v.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tW0djg1PYniApZbGL.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tWL-Yg1PYniApZbCs.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tWSdXg1PYniApZbDU.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tY6jwg1PYniApZbRY.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tav8Ag1PYniApZbbK.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tbcUdg1PYniApZbe1.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tc_xZg1PYniApZbnL.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0te0TCg1PYniApZbw9.json create mode 100644 auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tes4Eg1PYniApZbwV.json diff --git a/auditbeat/module/audit/_meta/kibana/5.x/dashboard/AV0tXkjYg1PYniApZbKP.json b/auditbeat/module/audit/_meta/kibana/5.x/dashboard/AV0tXkjYg1PYniApZbKP.json new file mode 100644 index 00000000000..d067342aea8 --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/dashboard/AV0tXkjYg1PYniApZbKP.json @@ -0,0 +1,13 @@ +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "Auditbeat - File Integrity", + "uiStateJSON": "{\"P-1\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-7\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-8\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-9\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "panelsJSON": "[{\"col\":1,\"id\":\"AV0tVcg6g1PYniApZa-v\",\"panelIndex\":1,\"row\":1,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":3,\"id\":\"AV0tV05vg1PYniApZbA2\",\"panelIndex\":2,\"row\":1,\"size_x\":7,\"size_y\":6,\"type\":\"visualization\"},{\"col\":10,\"id\":\"AV0tWL-Yg1PYniApZbCs\",\"panelIndex\":3,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"AV0tWSdXg1PYniApZbDU\",\"panelIndex\":4,\"row\":4,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AV0tW0djg1PYniApZbGL\",\"panelIndex\":5,\"row\":9,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV0tY6jwg1PYniApZbRY\",\"panelIndex\":6,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AV0tav8Ag1PYniApZbbK\",\"panelIndex\":7,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"AV0tbcUdg1PYniApZbe1\",\"panelIndex\":8,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":5,\"panelIndex\":9,\"type\":\"visualization\",\"id\":\"AV0tc_xZg1PYniApZbnL\",\"col\":1,\"row\":12},{\"size_x\":4,\"size_y\":3,\"panelIndex\":10,\"type\":\"visualization\",\"id\":\"AV0tes4Eg1PYniApZbwV\",\"col\":9,\"row\":9},{\"size_x\":4,\"size_y\":3,\"panelIndex\":11,\"type\":\"visualization\",\"id\":\"AV0te0TCg1PYniApZbw9\",\"col\":1,\"row\":9}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tV05vg1PYniApZbA2.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tV05vg1PYniApZbA2.json new file mode 100644 index 00000000000..a6f965c5176 --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tV05vg1PYniApZbA2.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Events over time\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per 5 minutes\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Action\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Events over time", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tVcg6g1PYniApZa-v.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tVcg6g1PYniApZa-v.json new file mode 100644 index 00000000000..14bba1b33f9 --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tVcg6g1PYniApZa-v.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Action Metrics\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":false,\"bgFill\":\"#000\",\"fontSize\":\"24\",\"labelColor\":false,\"subText\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":true,\"extendRange\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Actions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Action Metrics", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tW0djg1PYniApZbGL.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tW0djg1PYniApZbGL.json new file mode 100644 index 00000000000..dc98cc18d6c --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tW0djg1PYniApZbGL.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Top updated\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Top updated", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"audit.file.action:updated OR audit.file.action:attributes_modified\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tWL-Yg1PYniApZbCs.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tWL-Yg1PYniApZbCs.json new file mode 100644 index 00000000000..fbb618f9bae --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tWL-Yg1PYniApZbCs.json @@ -0,0 +1,10 @@ +{ + "visState": "{\n \"title\": \"Auditbeat - File - Top owners\",\n \"type\": \"pie\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"audit.file.owner\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\",\n \"customLabel\": \"Owner\"\n }\n }\n ],\n \"listeners\": {}\n}", + "description": "", + "title": "Auditbeat - File - Top owners", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"index\": \"auditbeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tWSdXg1PYniApZbDU.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tWSdXg1PYniApZbDU.json new file mode 100644 index 00000000000..880a4ea04a6 --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tWSdXg1PYniApZbDU.json @@ -0,0 +1,10 @@ +{ + "visState": "{\n \"title\": \"Auditbeat - File - Top groups\",\n \"type\": \"pie\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"audit.file.group\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\",\n \"customLabel\": \"Group\"\n }\n }\n ],\n \"listeners\": {}\n}", + "description": "", + "title": "Auditbeat - File - Top groups", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"index\": \"auditbeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tY6jwg1PYniApZbRY.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tY6jwg1PYniApZbRY.json new file mode 100644 index 00000000000..f1f8ec979e3 --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tY6jwg1PYniApZbRY.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Top agent by count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"23\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Top agent by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"beat.hostname\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Top agent by count", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:audit.file\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tav8Ag1PYniApZbbK.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tav8Ag1PYniApZbbK.json new file mode 100644 index 00000000000..7892f502a09 --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tav8Ag1PYniApZbbK.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Most changed file by count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"20\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Most changed file by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.path\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Most changed file by count", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"audit.file.type:file\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tbcUdg1PYniApZbe1.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tbcUdg1PYniApZbe1.json new file mode 100644 index 00000000000..a8f47a2346c --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tbcUdg1PYniApZbe1.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Most common mode by count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"20\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Most common mode by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.mode\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Most common mode by count", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tc_xZg1PYniApZbnL.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tc_xZg1PYniApZbnL.json new file mode 100644 index 00000000000..52423336cc5 --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tc_xZg1PYniApZbnL.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Event summary\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"beat.hostname\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Hostname\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"audit.file.path\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"audit.file.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Action\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Event summary", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0te0TCg1PYniApZbw9.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0te0TCg1PYniApZbw9.json new file mode 100644 index 00000000000..d33f45d4cae --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0te0TCg1PYniApZbw9.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Top created\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Top created", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"audit.file.action:created\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tes4Eg1PYniApZbwV.json b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tes4Eg1PYniApZbwV.json new file mode 100644 index 00000000000..c6d4545950f --- /dev/null +++ b/auditbeat/module/audit/_meta/kibana/5.x/visualization/AV0tes4Eg1PYniApZbwV.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Auditbeat - File - Top deleted\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}],\"listeners\":{}}", + "description": "", + "title": "Auditbeat - File - Top deleted", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"audit.file.action:deleted\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/testing/environments/5x.yml b/testing/environments/5x.yml index a45c8382d8c..843f3aed29b 100644 --- a/testing/environments/5x.yml +++ b/testing/environments/5x.yml @@ -3,7 +3,7 @@ version: '2.1' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:5.5.1 + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] environment: @@ -18,13 +18,13 @@ services: context: docker/logstash dockerfile: Dockerfile args: - ELASTIC_VERSION: 5.5.1 + ELASTIC_VERSION: 5.6.1 DOWNLOAD_URL: https://artifacts.elastic.co/downloads environment: - ES_HOST=elasticsearch kibana: - image: docker.elastic.co/kibana/kibana:5.5.1 + image: docker.elastic.co/kibana/kibana:5.6.1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5601"] retries: 6 From 57f67e750fdd1f3bdcdc3a89a1e4f561daa4206f Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 25 Sep 2017 10:51:11 +0200 Subject: [PATCH 41/96] Improve logging for Size changes (#5194) Right now in case the size changes of a file and it triggers reading a file again, in the debug log its not visibile what the new size is. This should make detecting issues like https://github.com/elastic/beats/issues/5179 easier. --- filebeat/prospector/log/prospector.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/prospector/log/prospector.go b/filebeat/prospector/log/prospector.go index a58690aa3ea..e0331c28c0b 100644 --- a/filebeat/prospector/log/prospector.go +++ b/filebeat/prospector/log/prospector.go @@ -449,7 +449,7 @@ func (p *Prospector) harvestExistingFile(newState file.State, oldState file.Stat // Resume harvesting of an old file we've stopped harvesting from // This could also be an issue with force_close_older that a new harvester is started after each scan but not needed? // One problem with comparing modTime is that it is in seconds, and scans can happen more then once a second - logp.Debug("prospector", "Resuming harvesting of file: %s, offset: %v", newState.Source, oldState.Offset) + logp.Debug("prospector", "Resuming harvesting of file: %s, offset: %d, new size: %d", newState.Source, oldState.Offset, newState.Fileinfo.Size()) err := p.startHarvester(newState, oldState.Offset) if err != nil { logp.Err("Harvester could not be started on existing file: %s, Err: %s", newState.Source, err) @@ -459,7 +459,7 @@ func (p *Prospector) harvestExistingFile(newState file.State, oldState file.Stat // File size was reduced -> truncated file if oldState.Finished && newState.Fileinfo.Size() < oldState.Offset { - logp.Debug("prospector", "Old file was truncated. Starting from the beginning: %s", newState.Source) + logp.Debug("prospector", "Old file was truncated. Starting from the beginning: %s, offset: %d, new size: %d ", newState.Source, newState.Fileinfo.Size()) err := p.startHarvester(newState, 0) if err != nil { logp.Err("Harvester could not be started on truncated file: %s, Err: %s", newState.Source, err) From f1bc7146afaf45dcd26f6cb8f5fc0d2ef07064f6 Mon Sep 17 00:00:00 2001 From: Tyler Clendenin Date: Mon, 25 Sep 2017 05:13:58 -0400 Subject: [PATCH 42/96] cms_get and cmd_set mixed up in memcached module (#5189) I just created this here and thought to look at the code. https://discuss.elastic.co/t/metricbeat-memcached-module-seems-to-mix-up-memcached-stats-cmd-get-and-memcached-stats-cmd-set/100732 --- metricbeat/module/memcached/stats/data.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/memcached/stats/data.go b/metricbeat/module/memcached/stats/data.go index 4cbd723cf5c..60c0c559fb9 100644 --- a/metricbeat/module/memcached/stats/data.go +++ b/metricbeat/module/memcached/stats/data.go @@ -21,8 +21,8 @@ var ( "misses": c.Int("get_misses"), }, "cmd": s.Object{ - "get": c.Int("cmd_set"), - "set": c.Int("cmd_get"), + "get": c.Int("cmd_get"), + "set": c.Int("cmd_set"), }, "read": s.Object{ "bytes": c.Int("bytes_read"), From 829f833153ed1d0b88da43b354888923ba87a431 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 25 Sep 2017 12:17:56 +0200 Subject: [PATCH 43/96] Added changelog entry for #5189 (#5231) --- CHANGELOG.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index cf55fc9daff..623a20159e5 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -44,7 +44,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di *Auditbeat* *Filebeat* -- Fix default paths for redis 4.0.1 logs on macOS {pull}5173[5173] +- Fix default paths for redis 4.0.1 logs on macOS {pull}5173[5173] *Heartbeat* @@ -58,6 +58,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Don't start metricbeat if external modules config is wrong and reload is disabled {pull}5053[5053] - Fix kubernetes events module to be able to index time fields properly. {issue}5093[5093] - The MongoDB module now connects on each fetch, to avoid stopping the whole Metricbeat instance if MongoDB is not up when starting. {pull}5120[5120] +- Fixed `cmd_set` and `cmd_get` being mixed in the Memcache module. {pull}5189[5189] *Packetbeat* From 10d0960e088aa050a4edfe71067558024e8057fe Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 25 Sep 2017 14:31:13 +0200 Subject: [PATCH 44/96] Update outputconfig.asciidoc (#5157) (#5232) Made it clear that we only support a single output config directive. (cherry picked from commit e4129700158291b9ddab9cdf8912ccf037ea4d6e) --- libbeat/docs/outputconfig.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index b30d126ac09..b2954bcd2d2 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -14,7 +14,8 @@ == Configure the output You configure {beatname_uc} to write to a specific output by setting options -in the `output` section of the +{beatname_lc}.yml+ config file. +in the `output` section of the +{beatname_lc}.yml+ config file. Only a single +output may be defined. The following topics describe how to configure each supported output: From 6f6a0720e4f99d8a37fce5c656f91ca788494fd4 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 25 Sep 2017 14:35:35 +0200 Subject: [PATCH 45/96] Add script to test against apm-server (#5099) apm-server depends on libbeat. To make sure we are aware when changes in libbeat break apm-server, this script allows to run the current code base against a specific branch / commit of apm-server. The idea is to add this also to our tests in a later PR but allow it to fail. Like this we always know when we break the code from apm-server. --- Makefile | 5 +++++ script/test_apm.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 script/test_apm.sh diff --git a/Makefile b/Makefile index 578d46609b1..b76016bb86a 100644 --- a/Makefile +++ b/Makefile @@ -140,3 +140,8 @@ notice: python-env python-env: @test -d $(PYTHON_ENV) || virtualenv $(VIRTUALENV_PARAMS) $(PYTHON_ENV) @$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8 six + +# Tests if apm works with the current code +.PHONY: python-env +test-apm: + sh ./script/test_apm.sh diff --git a/script/test_apm.sh b/script/test_apm.sh new file mode 100644 index 00000000000..11edeb8f3b7 --- /dev/null +++ b/script/test_apm.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# This script allows to test against the code from apm-server +# By default it is checked against master. The env variable BRANCH +# can be set to any value accepted by git checkout. +# +# go build is executed on the apm-server code to verify that it can +# still be built and runs the go tests with make unit. + +set -e + +# Check if a special branch env variable is set +if [ -z "${BRANCH}" ]; then + BRANCH=master +fi + +echo "apm-server branch: $BRANCH" + +BASE_PATH=$(pwd) +BUILD_DIR=$BASE_PATH/build +ELASTIC_DIR=$BUILD_DIR/apm-test/src/github.com/elastic +APM_SERVER_DIR=$ELASTIC_DIR/apm-server + +# Cleanup and create directories +rm -rf $APM_SERVER_DIR +mkdir -p $APM_SERVER_DIR + +# Clone and checkout defined branch +git clone https://github.com/elastic/apm-server $APM_SERVER_DIR +cd $APM_SERVER_DIR +git checkout $BRANCH +cd $BASE_PATH + +# Replace libbeat with local libbeat version +rm -r $APM_SERVER_DIR/vendor/github.com/elastic/beats/libbeat +cp -r libbeat $APM_SERVER_DIR/vendor/github.com/elastic/beats/ + +cd $APM_SERVER_DIR + +echo "Build apm-server binary" + +# Set temporary GOPATH to make sure local version of libbeat is used +GOPATH=$BUILD_DIR/apm-test go build + +echo "Run apm-server unit tests" +GOPATH=$BUILD_DIR/apm-test make unit From 5ecba0f3dbb0525b9740c686bd89a1740df25537 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Sep 2017 10:17:32 -0400 Subject: [PATCH 46/96] Add jenkins scripts to repo (#5235) This adds the scripts that are executed by Jenkins to this repo. They need to be added to other branches and kept in sync. --- dev-tools/jenkins_ci | 155 ------------------------------------ dev-tools/jenkins_ci.ps1 | 61 ++++++++++++++ dev-tools/jenkins_ci.sh | 42 ++++++++++ dev-tools/jenkins_intake.sh | 27 +++++++ 4 files changed, 130 insertions(+), 155 deletions(-) delete mode 100755 dev-tools/jenkins_ci create mode 100755 dev-tools/jenkins_ci.ps1 create mode 100755 dev-tools/jenkins_ci.sh create mode 100755 dev-tools/jenkins_intake.sh diff --git a/dev-tools/jenkins_ci b/dev-tools/jenkins_ci deleted file mode 100755 index 31af83fdd87..00000000000 --- a/dev-tools/jenkins_ci +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Script directory: -SDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -SNAME="$(basename "$0")" - -source "${SDIR}/common.bash" - -usage() { -cat << EOF - Usage: $SNAME [-d] [-h] [-v] [-r] [-w=WORKSPACE] (-g|-b|-c) - - Description: Executes a build using the project's Go version. - - Options: - -w=WORKSPACE Required. Specifies the path to the Jenkins workspace. - If not set then the WORKSPACE environment variable is - used. The workspace will be treated as the GOPATH. - -b | --build Perform a build which includes make targets: check, - testsuite, coverage-report, and docs. - -c | --cleanup Clean up after the build by removing the checkout of - elastic/docs and stopping any running containers - started by the build. This cannot be specified with - --build. - -g | --go-version Optional. Write the project's Go version to stdout - and then exits. Can be used to setup Go with - eval "\$(gimme \$(./jenkins_ci -g))". - -i | --install-gimme Optional. Installs gimme to HOME/bin. - -r | --race Optional. Enable the Go race detector for tests that - are run. - -d | --debug Optional. Runs the script with 'set -x' to log a trace - of all commands and their arguments being executed. - -v | --verbose Optional. Enable verbose logging from this script to stderr. - -h | --help Optional. Print this usage information. - - Examples: - Print project Go version: ./$SNAME --go-version - Build with race detector: ./$SNAME -b -r - Stop test environment: ./$SNAME -c - - Jenkins Setup: - - 1) Jenkins should be setup to checkout elastic/beats into - \$WORKSPACE/src/github.com/elastic/ - 2) The single build script should be added that executes - \$WORKSPACE/src/github.com/elastic/beats/dev-tools/$SNAME -d -v -b --race - 3) A post build action should be added that executes - \$WORKSPACE/src/github.com/elastic/beats/dev-tools/$SNAME -d -v -c -EOF -} - -# Parse command line arguments. -parse_args() { - for i in "$@" - do - case $i in - -b|--build) - BUILD=true - shift - ;; - -c|--cleanup) - CLEANUP=true - shift - ;; - -d|--debug) - set -x - shift - ;; - -g|--go-version) - get_go_version - echo "${GO_VERSION}" - exit 0 - ;; - -h|--help) - usage - exit 1 - ;; - -i|--install-gimme) - install_gimme - exit 0 - ;; - -r|--race) - export RACE_DETECTOR=1 - shift - ;; - -v|--verbose) - VERBOSE=true - shift - ;; - -w=*|--workspace=*) - WORKSPACE="${i#*=}" - shift - ;; - *) - echo "Invalid argument: $i" - usage - exit 1 - ;; - esac - done - - if [ -z "$WORKSPACE" ]; then - err "WORKSPACE env var must be set or --workspace must be specified" - exit 1 - fi -} - -lint() { - if [ -z $ghprbPullId ]; then - return - fi - export REVIEWDOG_OPTIONS="-ci common" - IFS='/' read -a repo <<< "$ghprbGhRepository" - CI_PULL_REQUEST=$ghprbPullId CI_COMMIT=$ghprbActualCommit CI_REPO_OWNER=${repo[0]} CI_REPO_NAME=${repo[1]} make lint -} - -build() { - make check - make testsuite - make coverage-report - make docs - lint -} - -cleanup() { - # Remove the checkout of elastic/docs if it exists. - rm -rf "${SDIR}/../build/docs" - - make stop-environments -} - -main() { - cd "${SDIR}/.." - parse_args $* - get_go_version - setup_go_root ${GO_VERSION} - setup_go_path ${WORKSPACE} - - if [ "$BUILD" == "true" ] && [ "$CLEANUP" == "true" ]; then - err "--build and --cleanup cannot be used together" - exit 1 - elif [ "$BUILD" == "true" ]; then - chmod -R go-w "${GOPATH}/src/github.com/elastic/beats" - build - elif [ "$CLEANUP" == "true" ]; then - cleanup - else - err "Use either --build or --cleanup" - exit 1 - fi -} - -umask 022 -main $* diff --git a/dev-tools/jenkins_ci.ps1 b/dev-tools/jenkins_ci.ps1 new file mode 100755 index 00000000000..a7f9f5d37a4 --- /dev/null +++ b/dev-tools/jenkins_ci.ps1 @@ -0,0 +1,61 @@ +function Exec +{ + param( + [Parameter(Position=0,Mandatory=1)][scriptblock]$cmd, + [Parameter(Position=1,Mandatory=0)][string]$errorMessage = ($msgs.error_bad_command -f $cmd) + ) + + & $cmd + if ($LastExitCode -ne 0) { + Write-Error $errorMessage + exit $LastExitCode + } +} + +# Setup Go. +$env:GOPATH = $env:WORKSPACE +$env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH" +if (Test-Path -PathType leaf .go-version) { + & gvm --format=powershell $(Get-Content .go-version) | Invoke-Expression +} else { + & gvm --format=powershell 1.7.5 | Invoke-Expression +} + +if (Test-Path "$env:beat") { + cd "$env:beat" +} else { + echo "$env:beat does not exist" + New-Item -ItemType directory -Path build | Out-Null + New-Item -Name build\TEST-empty.xml -ItemType File | Out-Null + exit +} + +if (Test-Path "build") { Remove-Item -Recurse -Force build } +New-Item -ItemType directory -Path build\coverage | Out-Null +New-Item -ItemType directory -Path build\system-tests | Out-Null +New-Item -ItemType directory -Path build\system-tests\run | Out-Null + +exec { go get -u github.com/jstemmer/go-junit-report } + +echo "Building $env:beat" +exec { go build } "Build FAILURE" + +if ($env:beat -eq "metricbeat") { + cp .\_meta\fields.common.yml .\_meta\fields.generated.yml + python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\_meta\fields.generated.yml +} elseif ($env:beat -eq "libbeat") { + cp .\_meta\fields.common.yml .\_meta\fields.generated.yml + cat processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath .\_meta\fields.generated.yml + cp .\_meta\fields.generated.yml .\fields.yml +} + +echo "Unit testing $env:beat" +go test -v $(go list ./... | select-string -Pattern "vendor" -NotMatch) 2>&1 | Out-File -encoding UTF8 build/TEST-go-unit.out +exec { Get-Content build/TEST-go-unit.out | go-junit-report.exe -set-exit-code | Out-File -encoding UTF8 build/TEST-go-unit.xml } "Unit test FAILURE" + +echo "System testing $env:beat" +# TODO (elastic/beats#5050): Use a vendored copy of this. +exec { go get github.com/docker/libcompose } +exec { go test -race -c -cover -covermode=atomic -coverpkg ./... } +exec { cd tests/system } +exec { nosetests --with-timer --with-xunit --xunit-file=../../build/TEST-system.xml } "System test FAILURE" \ No newline at end of file diff --git a/dev-tools/jenkins_ci.sh b/dev-tools/jenkins_ci.sh new file mode 100755 index 00000000000..087d8ae50b1 --- /dev/null +++ b/dev-tools/jenkins_ci.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euox pipefail + +: "${HOME:?Need to set HOME to a non-empty value.}" +: "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" +: "${beat:?Need to set beat to a non-empty value.}" + +if [ ! -d "$beat" ]; then + echo "$beat does not exist" + mkdir -p build + touch build/TEST-empty.xml + exit +fi + +# Setup Go. +export GOPATH=${WORKSPACE} +export PATH=${GOPATH}/bin:${PATH} +if [ -f ".go-version" ]; then + eval "$(gvm $(cat .go-version))" +else + eval "$(gvm 1.7.5)" +fi + +# Workaround for Python virtualenv path being too long. +TEMP_PYTHON_ENV=$(mktemp -d) +export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env" + +cleanup() { + echo "Running cleanup..." + rm -rf $TEMP_PYTHON_ENV + make stop-environment fix-permissions + echo "Killing all running containers..." + docker ps -q | xargs -r docker kill || true + echo "Cleaning stopped docker containers and dangling images/networks/volumes..." + docker system prune -f || true + echo "Cleanup complete." +} +trap cleanup EXIT + +rm -rf ${GOPATH}/pkg +cd ${beat} +RACE_DETECTOR=1 make clean check testsuite \ No newline at end of file diff --git a/dev-tools/jenkins_intake.sh b/dev-tools/jenkins_intake.sh new file mode 100755 index 00000000000..9e23ea6a325 --- /dev/null +++ b/dev-tools/jenkins_intake.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euox pipefail + +: "${HOME:?Need to set HOME to a non-empty value.}" +: "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" + +# Setup Go. +export GOPATH=${WORKSPACE} +export PATH=${GOPATH}/bin:${PATH} +if [ -f ".go-version" ]; then + eval "$(gvm $(cat .go-version))" +else + eval "$(gvm 1.7.5)" +fi + +# Workaround for Python virtualenv path being too long. +TEMP_PYTHON_ENV=$(mktemp -d) +export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env" + +cleanup() { + echo "Running cleanup..." + rm -rf $TEMP_PYTHON_ENV + echo "Cleanup complete." +} +trap cleanup EXIT + +make check \ No newline at end of file From f61ecc393f275aeb571d559dee91c8f2ac30d9a9 Mon Sep 17 00:00:00 2001 From: Gilad Peleg Date: Mon, 25 Sep 2017 18:00:15 +0300 Subject: [PATCH 47/96] Add .go files style to editorconfig (#5236) --- .editorconfig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.editorconfig b/.editorconfig index a92dc2185bd..bd0cc74b875 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,6 +7,10 @@ end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true +[.go] +indent_size = 4 +indent_style = tab + [*.json] indent_size = 4 indent_style = space From ebf641ed5f92406791fb86ae5f2c5be22212c3a0 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Sep 2017 18:10:23 -0400 Subject: [PATCH 48/96] Remove multiple output comment from config (#5193) Remove multiple output comment --- auditbeat/auditbeat.reference.yml | 3 +-- auditbeat/auditbeat.yml | 3 +-- filebeat/filebeat.reference.yml | 3 +-- filebeat/filebeat.yml | 3 +-- heartbeat/heartbeat.reference.yml | 3 +-- heartbeat/heartbeat.yml | 3 +-- libbeat/_meta/config.reference.yml | 3 +-- libbeat/_meta/config.yml | 3 +-- metricbeat/metricbeat.reference.yml | 3 +-- metricbeat/metricbeat.yml | 3 +-- packetbeat/packetbeat.reference.yml | 3 +-- packetbeat/packetbeat.yml | 3 +-- winlogbeat/winlogbeat.reference.yml | 3 +-- winlogbeat/winlogbeat.yml | 3 +-- 14 files changed, 14 insertions(+), 28 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index a44e8b11c2d..665125f9fc9 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -190,8 +190,7 @@ auditbeat.modules: #================================ Outputs ====================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- output.elasticsearch: diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml index 64e72e11a23..925366e2187 100644 --- a/auditbeat/auditbeat.yml +++ b/auditbeat/auditbeat.yml @@ -90,8 +90,7 @@ setup.kibana: #================================ Outputs ===================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 6c9f03a3ace..72a12bb38b6 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -610,8 +610,7 @@ filebeat.prospectors: #================================ Outputs ====================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- output.elasticsearch: diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 8e53432e054..4202604a534 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -137,8 +137,7 @@ setup.kibana: #================================ Outputs ===================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 73d3325b47f..5b8c2d04741 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -339,8 +339,7 @@ heartbeat.scheduler: #================================ Outputs ====================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- output.elasticsearch: diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml index ac4ba43ee04..fbea35a018e 100644 --- a/heartbeat/heartbeat.yml +++ b/heartbeat/heartbeat.yml @@ -84,8 +84,7 @@ setup.kibana: #================================ Outputs ===================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index be10313e7a8..5e8eaec312e 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -125,8 +125,7 @@ #================================ Outputs ====================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- output.elasticsearch: diff --git a/libbeat/_meta/config.yml b/libbeat/_meta/config.yml index d3847a08aec..6c94b731a64 100644 --- a/libbeat/_meta/config.yml +++ b/libbeat/_meta/config.yml @@ -54,8 +54,7 @@ setup.kibana: #================================ Outputs ===================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f8d138f6f1a..5dba70d55d8 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -569,8 +569,7 @@ metricbeat.modules: #================================ Outputs ====================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- output.elasticsearch: diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index ccab3b74fbc..fe852104792 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -81,8 +81,7 @@ setup.kibana: #================================ Outputs ===================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index b2efb313313..5cb8fadc656 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -577,8 +577,7 @@ packetbeat.protocols: #================================ Outputs ====================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- output.elasticsearch: diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index c491ed07f70..779daba2cb3 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -155,8 +155,7 @@ setup.kibana: #================================ Outputs ===================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 949abde3d9f..79b1b469c22 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -154,8 +154,7 @@ winlogbeat.event_logs: #================================ Outputs ====================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- output.elasticsearch: diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 0ab49f1c3bf..8db915b898e 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -85,8 +85,7 @@ setup.kibana: #================================ Outputs ===================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: From aaf34c81eed649c49893f33865976176b4b2cfd8 Mon Sep 17 00:00:00 2001 From: Monica Sarbu Date: Tue, 26 Sep 2017 10:29:21 +0200 Subject: [PATCH 49/96] Add yml file with the list of dashboards available for Auditbeat (#5182) Rename the dashboards and the visualizations to use the same format as the other Beats --- .../dashboard/auditbeat-file-integrity.json | 84 +++++++++---------- auditbeat/module/audit/module.yml | 3 + 2 files changed, 45 insertions(+), 42 deletions(-) create mode 100644 auditbeat/module/audit/module.yml diff --git a/auditbeat/module/audit/_meta/kibana/default/dashboard/auditbeat-file-integrity.json b/auditbeat/module/audit/_meta/kibana/default/dashboard/auditbeat-file-integrity.json index 64ef911145e..d98c56b9ef1 100644 --- a/auditbeat/module/audit/_meta/kibana/default/dashboard/auditbeat-file-integrity.json +++ b/auditbeat/module/audit/_meta/kibana/default/dashboard/auditbeat-file-integrity.json @@ -4,12 +4,12 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Action Metrics", + "title": "Actions [Auditbeat File]", "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", "version": 1, - "visState": "{\"title\":\"Auditbeat - File - Action Metrics\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":false,\"bgFill\":\"#000\",\"fontSize\":\"24\",\"labelColor\":false,\"subText\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":true,\"extendRange\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Actions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + "visState": "{\"title\":\"Actions [Auditbeat File]\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":false,\"bgFill\":\"#000\",\"fontSize\":\"24\",\"labelColor\":false,\"subText\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":true,\"extendRange\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Actions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}" }, "id": "AV0tVcg6g1PYniApZa-v", "type": "visualization", @@ -19,72 +19,72 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Events over time", + "title": "Events over time [Auditbeat File]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Auditbeat - File - Events over time\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per 5 minutes\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Action\"}}],\"listeners\":{}}" + "visState": "{\"title\":\"Events over time [Auditbeat File]\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per 5 minutes\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Action\"}}]}" }, "id": "AV0tV05vg1PYniApZbA2", "type": "visualization", - "version": 1 + "version": 2 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"index\": \"auditbeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Top owners", + "title": "Top owners [Auditbeat File]", "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"title\": \"Auditbeat - File - Top owners\",\n \"type\": \"pie\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"audit.file.owner\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\",\n \"customLabel\": \"Owner\"\n }\n }\n ],\n \"listeners\": {}\n}" + "visState": "{\"title\":\"Top owners [Auditbeat File]\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.owner\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Owner\"}}]}" }, "id": "AV0tWL-Yg1PYniApZbCs", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"index\": \"auditbeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Top groups", + "title": "Top groups [Auditbeat File]", "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"title\": \"Auditbeat - File - Top groups\",\n \"type\": \"pie\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"audit.file.group\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\",\n \"customLabel\": \"Group\"\n }\n }\n ],\n \"listeners\": {}\n}" + "visState": "{\"title\":\"Top groups [Auditbeat File]\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.group\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Group\"}}]}" }, "id": "AV0tWSdXg1PYniApZbDU", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"audit.file.action:updated OR audit.file.action:attributes_modified\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"audit.file.action:updated OR audit.file.action:attributes_modified\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Top updated", + "title": "Top updated [Auditbeat File]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Auditbeat - File - Top updated\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}],\"listeners\":{}}" + "visState": "{\"title\":\"Top updated [Auditbeat File]\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}]}" }, "id": "AV0tW0djg1PYniApZbGL", "type": "visualization", - "version": 4 + "version": 2 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:audit.file\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"_exists_:audit.file\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Top agent by count", + "title": "Top agent by count [Auditbeat File]", "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", "version": 1, - "visState": "{\"title\":\"Auditbeat - File - Top agent by count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"23\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Top agent by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"beat.hostname\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + "visState": "{\"title\":\"Top agent by count [Auditbeat File]\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"23\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Top agent by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"beat.hostname\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}]}" }, "id": "AV0tY6jwg1PYniApZbRY", "type": "visualization", @@ -94,31 +94,31 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"audit.file.type:file\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"audit.file.type:file\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Most changed file by count", + "title": "Most changed file by count [Auditbeat File]", "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", "version": 1, - "visState": "{\"title\":\"Auditbeat - File - Most changed file by count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"20\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Most changed file by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.path\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + "visState": "{\"title\":\"Most changed file by count [Auditbeat File]\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"20\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Most changed file by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.path\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}]}" }, "id": "AV0tav8Ag1PYniApZbbK", "type": "visualization", - "version": 1 + "version": 2 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Most common mode by count", + "title": "Most common mode by count [Auditbeat File]", "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", "version": 1, - "visState": "{\"title\":\"Auditbeat - File - Most common mode by count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"20\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Most common mode by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.mode\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}" + "visState": "{\"title\":\"Most common mode by count [Auditbeat File]\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"20\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Most common mode by count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit.file.mode\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\"}}]}" }, "id": "AV0tbcUdg1PYniApZbe1", "type": "visualization", - "version": 1 + "version": 2 }, { "attributes": { @@ -139,12 +139,12 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"audit.file.action:deleted\"}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"audit.file.action:deleted\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Top deleted", + "title": "Top deleted [Auditbeat File]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Auditbeat - File - Top deleted\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}],\"listeners\":{}}" + "visState": "{\"title\":\"Top deleted [Auditbeat File]\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"legendPosition\":\"right\",\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}]}" }, "id": "AV0tes4Eg1PYniApZbwV", "type": "visualization", @@ -154,12 +154,12 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query_string\":{\"query\":\"audit.file.action:created\",\"analyze_wildcard\":true}},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"auditbeat-*\",\"query\":{\"query\":{\"query_string\":{\"query\":\"audit.file.action:created\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, - "title": "Auditbeat - File - Top created", + "title": "Top created [Auditbeat File]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Auditbeat - File - Top created\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}],\"listeners\":{}}" + "visState": "{\"title\":\"Top created [Auditbeat File]\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"legendPosition\":\"right\",\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit.file.path\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Path\"}}]}" }, "id": "AV0te0TCg1PYniApZbw9", "type": "visualization", @@ -167,22 +167,22 @@ }, { "attributes": { - "description": "", + "description": "Monitor file changes", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}}}" }, "optionsJSON": "{\"darkTheme\":false}", - "panelsJSON": "[{\"col\":1,\"id\":\"AV0tVcg6g1PYniApZa-v\",\"panelIndex\":1,\"row\":1,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":3,\"id\":\"AV0tV05vg1PYniApZbA2\",\"panelIndex\":2,\"row\":1,\"size_x\":7,\"size_y\":6,\"type\":\"visualization\"},{\"col\":10,\"id\":\"AV0tWL-Yg1PYniApZbCs\",\"panelIndex\":3,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"AV0tWSdXg1PYniApZbDU\",\"panelIndex\":4,\"row\":4,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AV0tW0djg1PYniApZbGL\",\"panelIndex\":5,\"row\":9,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV0tY6jwg1PYniApZbRY\",\"panelIndex\":6,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AV0tav8Ag1PYniApZbbK\",\"panelIndex\":7,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"AV0tbcUdg1PYniApZbe1\",\"panelIndex\":8,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":5,\"panelIndex\":9,\"type\":\"visualization\",\"id\":\"AV0tc_xZg1PYniApZbnL\",\"col\":1,\"row\":12},{\"size_x\":4,\"size_y\":3,\"panelIndex\":10,\"type\":\"visualization\",\"id\":\"AV0tes4Eg1PYniApZbwV\",\"col\":9,\"row\":9},{\"size_x\":4,\"size_y\":3,\"panelIndex\":11,\"type\":\"visualization\",\"id\":\"AV0te0TCg1PYniApZbw9\",\"col\":1,\"row\":9}]", + "panelsJSON": "[{\"col\":1,\"id\":\"AV0tVcg6g1PYniApZa-v\",\"panelIndex\":1,\"row\":1,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":3,\"id\":\"AV0tV05vg1PYniApZbA2\",\"panelIndex\":2,\"row\":1,\"size_x\":7,\"size_y\":6,\"type\":\"visualization\"},{\"col\":10,\"id\":\"AV0tWL-Yg1PYniApZbCs\",\"panelIndex\":3,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"AV0tWSdXg1PYniApZbDU\",\"panelIndex\":4,\"row\":4,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AV0tW0djg1PYniApZbGL\",\"panelIndex\":5,\"row\":9,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV0tY6jwg1PYniApZbRY\",\"panelIndex\":6,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AV0tav8Ag1PYniApZbbK\",\"panelIndex\":7,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"AV0tbcUdg1PYniApZbe1\",\"panelIndex\":8,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV0tc_xZg1PYniApZbnL\",\"panelIndex\":9,\"row\":12,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":9,\"id\":\"AV0tes4Eg1PYniApZbwV\",\"panelIndex\":10,\"row\":9,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AV0te0TCg1PYniApZbw9\",\"panelIndex\":11,\"row\":9,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"}]", "timeRestore": false, - "title": "Auditbeat - File Integrity", + "title": "[Auditbeat File] File Integrity", "uiStateJSON": "{\"P-1\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-7\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-8\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-9\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", "version": 1 }, "id": "AV0tXkjYg1PYniApZbKP", "type": "dashboard", - "version": 7 + "version": 3 } ], - "version": "6.0.0-alpha3-SNAPSHOT" + "version": "6.0.0-beta2" } \ No newline at end of file diff --git a/auditbeat/module/audit/module.yml b/auditbeat/module/audit/module.yml new file mode 100644 index 00000000000..8d02d7c1c63 --- /dev/null +++ b/auditbeat/module/audit/module.yml @@ -0,0 +1,3 @@ +dashboards: +- id: AV0tXkjYg1PYniApZbKP + file: Auditbeat-file-integrity.json From 0b104ff0f0fb82e0870fd50e78b07fe3c3d7193a Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Wed, 27 Sep 2017 01:08:09 -0700 Subject: [PATCH 50/96] Add fix implemented in #4845 (#5253) --- libbeat/docs/processors-using.asciidoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index e655cd5e2d9..22264095793 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -410,7 +410,7 @@ processors: fields: ["field1", "field2", ...] process_array: false max_depth: 1 - target: + target: "" overwrite_keys: false ----------------------------------------------------- @@ -423,7 +423,8 @@ arrays. The default is false. `target`:: (Optional) The field under which the decoded JSON will be written. By default the decoded JSON object replaces the string field from which it was read. To merge the decoded JSON fields into the root of the event, specify -`target` with an empty value (`target:`). +`target` with an empty string (`target: ""`). Note that the `null` value (`target:`) +is treated as if the field was not set at all. `overwrite_keys`:: (Optional) A boolean that specifies whether keys that already exist in the event are overwritten by keys from the decoded JSON object. The default value is false. From 94b62981f2690a5faf1281f8d93fdb3d33d237ef Mon Sep 17 00:00:00 2001 From: Monica Sarbu Date: Wed, 27 Sep 2017 13:11:53 +0200 Subject: [PATCH 51/96] Update import/export dashboards section (#5229) --- docs/devguide/newdashboards.asciidoc | 356 +++++++++++++-------------- 1 file changed, 172 insertions(+), 184 deletions(-) diff --git a/docs/devguide/newdashboards.asciidoc b/docs/devguide/newdashboards.asciidoc index df336935013..8757b0dc1a7 100644 --- a/docs/devguide/newdashboards.asciidoc +++ b/docs/devguide/newdashboards.asciidoc @@ -1,17 +1,18 @@ [[new-dashboards]] -== Creating New Kibana Dashboards for a Beat +== Creating New Kibana Dashboards for a Beat or a Beat module -NOTE: Starting with Beats 5.0.0, the Kibana dashboards are not released as part of the Beat package. They are released in a separate -package called `beats-dashboards`. +++++ +Creating New Kibana Dashboards +++++ -When contributing to Beats development, you may want to add new dashboards or modify existing ones. To make this easier, -we've created an `import_dashboards` script that you can use to <> from an -existing Beat into Kibana, where you can modify the dashboards or use them as a starting point to create new dashboards. -Kibana saves the dashboards, along with all the dependencies such as visualizations, searches, and index patterns, in -a special index in Elasticsearch. By default the index is `.kibana`, but you can specify a different index. - -When you're done making changes to the dashboards in Kibana, you can use the `export_dashboards` script to <>, along with all dependencies, to a local directory. +When contributing to Beats development, you may want to add new dashboards or +customize the existing ones. To get started, you can +<> that come with the official +Beats and use them as a starting point for your own dashboards. When you're done +making changes to the dashboards in Kibana, you can use the `export_dashboards` +script to <>, along with all +dependencies, to a local directory. To make sure the dashboards are compatible with the latest version of Kibana and Elasticsearch, we recommend that you use the virtual environment under @@ -30,103 +31,111 @@ The following topics provide more detail about importing and working with Beats [[import-dashboards]] === Importing Existing Beat Dashboards -You can use the `import_dashboards` script to import all the dashboards and the index pattern for a Beat, including the dependencies such as visualizations and searches. -The `import_dashboards` script is available under -https://github.com/elastic/beats/tree/master/dev-tools/cmd/import_dashboards[beats/dev-tools/cmd/import_dashboards]. It's also available in each Beat package under the `scripts` directory. +The official Beats come with Kibana dashboards, and starting with 6.0.0, they +are part of every Beat package. You can use the Beat executable to import all +the dashboards and the index pattern for a Beat, including the dependencies +such as visualizations and searches. + +You can use the Beat executable to import all the dashboards and the index pattern for a Beat, including the dependencies such as visualizations and searches. + +To import the dashboards, run the `setup` command. + + +[source,shell] +------------------------- +./metricbeat setup +------------------------- + +The `setup` phase loads: + +- Index mapping template in Elasticsearch +- Kibana dashboards +- Machine Learning jobs (if available) + +For more details about the `setup` command, run the following: + +[source,shell] +------------------------- +./metricbeat help setup + +This command does initial setup of the environment: + + * Index mapping template in Elasticsearch to ensure fields are mapped. + * Kibana dashboards (where available). + * ML jobs (where available). -There are a couple of common use cases for importing dashboards: +Usage: + filebeat setup [flags] -* Users who are getting started with Beats may want to <>. -* Community Beats developers may want to <> to use as a starting point for new dashboards. +Flags: + --dashboards Setup dashboards only + -h, --help help for setup + --machine-learning Setup machine learning job configurations only + --modules string List of enabled modules (comma separated) + --template Setup index template only +--------------------------- -[[import-single-beat]] -==== Import Dashboards and/or the Index Pattern for a Single Beat +The flags are useful when you don't want to load everything. For example, to +import only the dashboards, use the `--dashboards` flag: -Using the `import_dashboards` script from the Beat package, you can import the dashboards and the index pattern to -Elasticsearch running on localhost for a single Beat (eg. Metricbeat): +[source,shell] +--------------------- +./metricbeat setup --dashboards +------------------------------- + +Starting with Beats 6.0.0, the dashboards are no longer loaded directly into Elasticsearch. Instead, they are imported directly into Kibana. +Thus, if your Kibana instance is not listening on localhost, or you enabled +X-Pack for Kibana, you need to either configure the Kibana endpoint in +the config for the Beat, or pass the Kibana host and credentials as +arguments to the `setup` command. For example: + +[source,shell] +------------------------- +./metricbeat setup -E setup.kibana.host=192.168.3.206:5601 -E setup.kibana.username=elastic -E setup.kibana.password=secret +-------------------------- + +By default, the `setup` command imports the dashboards from the `kibana` +directory, which is available in the Beat package. + +NOTE: The format of the saved dashboards is not compatible between Kibana 5.x and 6.x. Thus, the Kibana 5.x dashboards are available in +the `5.x` directory, and the Kibana 6.0 dashboards, and older are in the `default` directory. + +In case you are using customized dashboards, you can import them: - from a local directory: + [source,shell] ---------------------------------------------------------------------- -./scripts/import_dashboards -dir kibana/metricbeat +./metricbeat setup -E setup.dashboards.directory=kibana ---------------------------------------------------------------------- - from a local zip archive: + [source,shell] ---------------------------------------------------------------------- -./scripts/import_dashboards -file metricbeat-dashboards-1.1.zip +./metricbeat setup -E setup.dashboards.file=metricbeat-dashboards-6.0.zip ---------------------------------------------------------------------- -- from the official zip archive available at {dashboards}: +- from a zip archive available online: + [source,shell] ---------------------------------------------------------------------- -./scripts/import_dashboards +./metricbeat setup -E setup.dashboards.url=path/to/url ---------------------------------------------------------------------- + -See <> for more details about the structure of dashboard archives. -- from a zip archive available online: -+ -[source,shell] ------------------------ -./scripts/import_dashboards -url https://github.com/monicasarbu/metricbeat-dashboards/archive/v1.1.zip ------------------------ - -If you don't specify the location of the archive, then by default it's set to the official zip archive containing the index pattern and the dashboards of the official Beats. +See <> for a description of the `setup.dashboards` configuration options. -To import only the index-pattern for a single Beat (eg. Metricbeat) use: -[source,shell] ------------------------ -./scripts/import_dashboards -only-index ------------------------ - -To import only the dashboards together with visualizations and searches for a single Beat (eg. Metricbeat) use: - -[source,shell] ------------------------ -./scripts/import_dashboards -only-dashboards ------------------------ - - -NOTE: When running the `import_dashboards` script from within the Beat package, the `-beat` option is set automatically to the Beat -name. - -See <> for a description of other import options. [[import-dashboards-for-development]] ==== Import Dashboards for Development -For development or community Beats, it's easier to run the `import_dashboards` script from the https://github.com/elastic/beats/tree/master/dev-tools/cmd/import_dashboards[beats/dev-tools/cmd/import_dashboards] directory. In this case, you need to first compile the script: - -[source,shell] ------------------------ -cd beats/dev-tools/cmd/import_dashboards -make ------------------------ - -And then you can import the index pattern and the dashboards together with visualizations and searches for a single -Beat, by passing the `-beat` option. - -For example, to import the Metricbeat dashboards together with visualizations, -searches, and the Metricbeat index pattern: - -[source,shell] ------------------ -beats/dev-tools/cmd/import_dashboards/import_dashboards -beat metricbeat ------------------ - -For this example, you must specify `-beat metricbeat`. If the `-beat` option is not -specified, the script imports the dashboards of all Beats. - -NOTE: You can make use of the Makefile from the Beat GitHub repository to import the +You can make use of the Makefile from the Beat GitHub repository to import the dashboards. If Elasticsearch is running on localhost, then you can run the following command from the Beat repository: [source,shell] -------------------------------- -make -C import-dashboards +make import-dashboards -------------------------------- If Elasticsearch is running on a different host, then you can use the `ES_URL` variable: @@ -137,120 +146,42 @@ ES_URL="http://192.168.3.206:9200" make import-dashboards ------------------------------- [[import-dashboard-options]] -==== Command Line Options +==== Kibana dashboards configuration -The `import_dashboards` script accepts the following command-line options. To see all the available options, read the descriptions below or run: +The configuration file (`*.reference.yml`) of each Beat contains the `setup.dashboards` section for configuring from where to get the Kibana dashboards, as well as the name of the index pattern. +Each of these configuration options can be overwritten with the command line options by using `-E` flag. -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -./import_dashboards -h ----------------------------------------------------------------------- - -*`-beat `*:: -The Beat name. The Beat name is required when importing from a zip archive. When using `import_dashboards` from the Beat package, this option is set automatically with the name of -the Beat. When running the script from source, the default value is "", so you need to set this option in order to install the index pattern and -the dashboards for a single Beat. Otherwise the script imports the index pattern and the dashboards for all Beats. -*`-cacert `*:: -The Certificate Authority to use for server verification. +*`setup.dashboards.directory=`*:: +Local directory that contains the saved dashboards and their dependencies. +The default value is the `kibana` directory available in the Beat package. -*`-cert `*:: -The certificate to use for SSL client authentication. The certificate must be in -PEM format. +*`setup.dashboards.file=`*:: +Local zip archive with the dashboards. The archive can contain Kibana dashboards for a single Beat or for multiple Beats. The dashboards of each Beat are placed under a separate directory with the name of the Beat. -*`-dir `*:: -Local directory that contains the subdirectories: dashboard, visualization, search, and index-pattern. The default value is the current directory. - -*`-es `*:: -The Elasticsearch URL. The default value is http://localhost:9200. - -*`-file `*:: -Local zip archive with the dashboards. The archive can contain Kibana dashboards for a single Beat or for multiple Beats. +*`setup.dashboards.url=`*:: +Zip archive with the dashboards, available online. The archive can contain Kibana dashboards for a single Beat or for +multiple Beats. The dashboards for each Beat are placed under a separate directory with the name of the Beat. -*`-i `*:: +*`setup.dashboards.index `*:: You should only use this option if you want to change the index pattern name that's used by default. For example, if the default is `metricbeat-*`, you can change it to `custombeat-*`. -*`-insecure`*:: -If specified, "insecure" SSL connections are allowed. - -*`-k `*:: -The Elasticsearch index pattern where Kibana saves its configuration. The default value is `.kibana`. - -*`-key `*:: -The client certificate key. The key must be in PEM format. - -*`-only-dashboards`*:: -If specified, then only the dashboards, along with their visualizations and searches, are imported. The index pattern is -not imported. By default, this is false. - -*`-only-index`*:: -If specified, then only the index pattern is imported. The dashboards, along with their visualizations and searches, are not imported. By default, this is false. - -*`-pass `*:: -The password for authenticating the connection to Elasticsearch by using Basic Authentication. By default no username and password are used. - -*`-snapshot`*:: -Using `-snapshot` will import the snapshot dashboards build for the current version. This is mainly useful when running a snapshot Beat build for testing purpose. -+ -NOTE: When using `-snapshot`, `-url` will be ignored. - -*`-url `*:: -Zip archive with the dashboards, available online. The archive can contain Kibana dashboards for a single Beat or for -multiple Beats. - -*`-user `*:: -The username for authenticating the connection to Elasticsearch by using Basic Authentication. By default no username and password are used. - - -[[dashboards-archive-structure]] -==== Structure of the Dashboards Archive - -The zip archive contains dashboards for at least one Beat. The index pattern, dashboards, visualizations and searches -are available in a separate directory for each Beat, having the name of the Beat. For example the official zip archive (beats-dashboards-{stack-version}) has the following structure: - -[source,shell] -------------------------- - metricbeat/ - dashboard/ - search/ - visualization/ - index-pattern/ - packetbeat/ - dashboard/ - search/ - visualization/ - index-pattern/ - filebeat/ - index-pattern/ - winlogbeat/ - dashboard/ - search/ - visualization/ - index-pattern/ ------------------------- - [[build-dashboards]] === Building Your Own Beat Dashboards -For visualizing the dashboards of a Beat in Kibana you need to have configured: +NOTE: If you want to modify a dashboard that comes with a Beat, it's better to modify a copy of the dashboard because the Beat overwrites the dashboards during the setup phase in order to have the latest version. For duplicating a dashboard, just use the `Clone` button from the top of the page. -* the Beat index pattern, which specifies how Kibana should display the Beat fields -* the Beat dashboards, including the dependencies such as visualizations and searches - -For the Elastic Beats, the index pattern is available in the GitHub repository of each Beat under -`etc/kibana/index-pattern` or under the `beats-dashboards` zip archive, available for each Beat release. -For the community Beats, you can easily generate the index-pattern from the `etc/fields.yml` file. For more details -check the <> section. +Before building your own dashboards or customizing the existing ones, you need to load: -If you would like to build dashboards from scratch for any Elastic Beats, you can start by importing the same version of the index pattern as your Beat: +* the Beat index pattern, which specifies how Kibana should display the Beat fields +* the Beat dashboards that you want to customize -[source,shell] ---------------- -$ scripts/import_dashboards -only-index ---------------- +For the Elastic Beats, the index pattern is available in the Beat package under +`kibana/*/index-pattern`. The index-pattern is automatically generated from the `fields.yml` file, available in the Beat package. For more details +check the <> section. After creating your own dashboards in Kibana, you can <> to a local directory, and then <> in order to be able to share the dashboards with the community. @@ -258,15 +189,15 @@ directory, and then <> in order to be [[generate-index-pattern]] === Generating the Beat Index Pattern -If you change the fields exported by the Beat, you need to generate a new index pattern for your Beat. Otherwise -you can just use the index pattern available under the `etc/kibana/index-pattern` directory or in the `beats-dashboards` -archive for the Elastic Beats. +The index-pattern defines the format of each field, and it's used by Kibana to know how to display the field. +If you change the fields exported by the Beat, you need to generate a new index pattern for your Beat. Otherwise, you can just use the index pattern available under the `kibana/*/index-pattern` directory. -The Beat index pattern is generated from the `etc/fields.yml`, where all the fields for a Beat are defined. For each field, besides the `type`, you can configure the +The Beat index pattern is generated from the `fields.yml`, which contains all +the fields exported by the Beat. For each field, besides the `type`, you can configure the `format` field. The format informs Kibana about how to display a certain field. A good example is `percentage` or `bytes` to display fields as `50%` or `5MB`. -To generate the index pattern from the `etc/fields.yml`, you need to run the following command in the Beat repository: +To generate the index pattern from the `fields.yml`, you need to run the following command in the Beat repository: [source,shell] --------------- @@ -277,11 +208,13 @@ make update === Exporting New and Modified Beat Dashboards To export all the dashboards for any Elastic Beat or any community Beat, including any new or modified dashboards and all dependencies such as -visualizations, searches, you can use the Python script `export_dashboards.py` from -https://github.com/elastic/beats/tree/master/dev-tools[dev-tools]. See the dev-tools +visualizations, searches, you can use the Golang script `export_dashboards.go` from +https://github.com/elastic/beats/tree/master/dev-tools/cmd/dashboards[dev-tools] for exporting Kibana 6.0 dashboards or later, and the Python script `export_5x_dashboards.py` +for exporting Kibana 5.x dashboards. See the dev-tools https://github.com/elastic/beats/tree/master/dev-tools/README.md[readme] for more info. - +///////////////////// +NOT YET IMPLEMENTED NOTE: You can make use of the Makefile from the Beat GitHub repository to export all the Kibana dashboards for a Beat from your Elasticsearch. If Elasticsearch is running on localhost, then you just need to run the following command from the Beat repository: @@ -297,6 +230,61 @@ If Elasticsearch is running on a different host, then you can use the `ES_URL` v ES_URL="http://192.168.3.206:9200" make export-dashboards ---------------------------- +///////////////////// + +==== Exporting Kibana 6.0 dashboards and newer + +The `dev-tools/cmd/export_dashboards.go` script helps you export your customized Kibana 6.0 dashboards and newer. You might need to export a single dashboard or all the dashboards available for a module or Beat. + + +===== Export a single Kibana dashboard + +You can export a single dashboard by passing the dashboard ID in the `-dashboard` flag. + +NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is +`app/kibana#/dashboard/7fea2930-478e-11e7-b1f0-cb29bac6bf8b?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `7fea2930-478e-11e7-b1f0-cb29bac6bf8b`. + +[source,shell] +--------------- +cd filebeat/module/redis/_meta/kibana/default/dashboard +go run ../../../../../../../dev-tools/cmd/dashboards/export_dashboards.go -dashboard 7fea2930-478e-11e7-b1f0-cb29bac6bf8b -output Filebeat-redis.json +--------------- + +This generates the `Filebeat-redis.json` file that contains the dashboard for the Redis module of Filebeat, including the dependencies (visualizations and searches). + +===== Export all module/Beat dashboards + +Each module should contain a `module.yml` file with a list of all the dashboards available for the module. For the Beats that don't have support for modules (e.g. Packetbeat), +there is a `dashboards.yml` file that defines all the Packetbeat dashboards. + +Below, it's an example of the `module.yml` file for the system module in Metricbeat: + +[source,shell] +--------------- +dashboards: +- id: Metricbeat-system-overview + file: Metricbeat-system-overview.json + +- id: 79ffd6e0-faa0-11e6-947f-177f697178b8 + file: Metricbeat-host-overview.json + +- id: CPU-slash-Memory-per-container + file: Metricbeat-containers-overview.json +--------------- + + +Each dashboard is defined by an `id` and the name of json `file` where the dashboard is saved locally. + +By passing the yml file to the `export_dashboards.go` script, you can export all the dashboards defined: + +[source,shell] +------------------- +go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system/module.yml +------------------- + + + +==== Exporting Kibana 5.x dashboards To export only some Kibana dashboards for an Elastic Beat or community Beat, you can simply pass a regular expression to the `export_dashboards.py` script to match the selected Kibana dashboards. @@ -313,14 +301,14 @@ For example, to export all Kibana dashboards that start with the **Packetbeat** [source,shell] ---------------------------------------------------------------------- -python ../dev-tools/export_dashboards.py --regex Packetbeat* +python ../dev-tools/cmd/dashboards/export_dashboards.py --regex Packetbeat* ---------------------------------------------------------------------- To see all the available options, read the descriptions below or run: [source,shell] ---------------------------------------------------------------------- -python ../dev-tools/export_dashboards.py -h +python ../dev-tools/cmd/dashboards/export_dashboards.py -h ---------------------------------------------------------------------- *`--url `*:: @@ -349,7 +337,7 @@ output/ [[archive-dashboards]] === Archiving Your Beat Dashboards -The Kibana dashboards for the Elastic Beats are saved under the `etc/kibana` directory. To create a zip archive with the +The Kibana dashboards for the Elastic Beats are saved under the `kibana` directory. To create a zip archive with the dashboards, including visualizations and searches and the index pattern, you can run the following command in the Beat repository: @@ -359,7 +347,7 @@ make package-dashboards -------------- The Makefile is part of libbeat, which means that community Beats contributors can use the commands shown here to -archive dashboards. The dashboards must be available under the `etc/kibana` directory. +archive dashboards. The dashboards must be available under the `kibana` directory. Another option would be to create a repository only with the dashboards, and use the GitHub release functionality to create a zip archive. From 71bfaaabd66be00a1e628f0c47654c7f08824649 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Wed, 27 Sep 2017 15:01:24 +0200 Subject: [PATCH 52/96] Basic RabbitMQ queues metricset (#4788) Basic RabbitMQ queue metricset Related to #3887 --- CHANGELOG.asciidoc | 2 + metricbeat/Makefile | 2 +- metricbeat/docs/fields.asciidoc | 147 ++++++++++++++++++ metricbeat/docs/modules/rabbitmq.asciidoc | 6 +- .../docs/modules/rabbitmq/queue.asciidoc | 19 +++ metricbeat/include/list.go | 1 + metricbeat/metricbeat.reference.yml | 2 +- metricbeat/module/rabbitmq/_meta/config.yml | 2 +- .../_meta/testdata/queue_sample_response.json | 145 +++++++++++++++++ .../module/rabbitmq/node/_meta/data.json | 134 ++++++++++++++-- .../module/rabbitmq/queue/_meta/data.json | 61 ++++++++ .../module/rabbitmq/queue/_meta/docs.asciidoc | 3 + .../module/rabbitmq/queue/_meta/fields.yml | 75 +++++++++ metricbeat/module/rabbitmq/queue/data.go | 79 ++++++++++ .../rabbitmq/queue/node_integration_test.go | 60 +++++++ metricbeat/module/rabbitmq/queue/node_test.go | 76 +++++++++ metricbeat/module/rabbitmq/queue/queue.go | 55 +++++++ metricbeat/modules.d/rabbitmq.yml.disabled | 2 +- 18 files changed, 851 insertions(+), 20 deletions(-) create mode 100644 metricbeat/docs/modules/rabbitmq/queue.asciidoc create mode 100644 metricbeat/module/rabbitmq/_meta/testdata/queue_sample_response.json create mode 100644 metricbeat/module/rabbitmq/queue/_meta/data.json create mode 100644 metricbeat/module/rabbitmq/queue/_meta/docs.asciidoc create mode 100644 metricbeat/module/rabbitmq/queue/_meta/fields.yml create mode 100644 metricbeat/module/rabbitmq/queue/data.go create mode 100644 metricbeat/module/rabbitmq/queue/node_integration_test.go create mode 100644 metricbeat/module/rabbitmq/queue/node_test.go create mode 100644 metricbeat/module/rabbitmq/queue/queue.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 623a20159e5..0b5ec119f1a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -100,6 +100,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add http server metricset to support push metrics via http. {pull}4770[4770] - Make config object public for graphite and http server {pull}4820[4820] - Add system uptime metricset. {issue}[4848[4848] +- Add `filesystem.ignore_types` to system module for ignoring filesystem types. {issue}4685[4685] +- Add experimental `queue` metricset to RabbitMQ module. {pull}4788[4788] *Packetbeat* diff --git a/metricbeat/Makefile b/metricbeat/Makefile index 014408c997a..4447e80519c 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -27,7 +27,7 @@ kibana: # Collects all module and metricset fields .PHONY: fields -fields: +fields: python-env @mkdir -p _meta @cp ${ES_BEATS}/metricbeat/_meta/fields.common.yml _meta/fields.generated.yml @${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 8ce16fa9a43..3a8ce9d8277 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -7852,6 +7852,153 @@ type: long Node uptime. +[float] +== queue fields + +queue + + + +[float] +=== `rabbitmq.queue.name` + +type: keyword + +The name of the queue with non-ASCII characters escaped as in C. + + +[float] +=== `rabbitmq.queue.vhost` + +type: keyword + +Virtual host name with non-ASCII characters escaped as in C. + + +[float] +=== `rabbitmq.queue.durable` + +type: boolean + +Whether or not the queue survives server restarts. + + +[float] +=== `rabbitmq.queue.auto_delete` + +type: boolean + +Whether the queue will be deleted automatically when no longer used. + + +[float] +=== `rabbitmq.queue.exclusive` + +type: boolean + +Whether the queue is exclusive (i.e. has owner_pid). + + +[float] +=== `rabbitmq.queue.node` + +type: keyword + +Node name. + + +[float] +=== `rabbitmq.queue.state` + +type: keyword + +The state of the queue. Normally 'running', but may be "{syncing, MsgCount}" if the queue is synchronising. Queues which are located on cluster nodes that are currently down will be shown with a status of 'down'. + + +[float] +=== `rabbitmq.queue.arguments.max_priority` + +type: long + +Maximum number of priority levels for the queue to support. + + +[float] +=== `rabbitmq.queue.consumers.count` + +type: long + +Number of consumers. + + +[float] +=== `rabbitmq.queue.consumers.utilisation.pct` + +type: long + +format: percentage + +Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count. + + +[float] +=== `rabbitmq.queue.messages.total.count` + +type: long + +Sum of ready and unacknowledged messages (queue depth). + + +[float] +=== `rabbitmq.queue.messages.ready.count` + +type: long + +Number of messages ready to be delivered to clients. + + +[float] +=== `rabbitmq.queue.messages.unacknowledged.count` + +type: long + +Number of messages delivered to clients but not yet acknowledged. + + +[float] +=== `rabbitmq.queue.messages.persistent.count` + +type: long + +Total number of persistent messages in the queue (will always be 0 for transient queues). + + +[float] +=== `rabbitmq.queue.memory.bytes` + +type: long + +format: bytes + +Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures. + + +[float] +=== `rabbitmq.queue.disk.reads.count` + +type: long + +Total number of times messages have been read from disk by this queue since it started. + + +[float] +=== `rabbitmq.queue.disk.writes.count` + +type: long + +Total number of times messages have been written to disk by this queue since it started. + + [[exported-fields-redis]] == Redis fields diff --git a/metricbeat/docs/modules/rabbitmq.asciidoc b/metricbeat/docs/modules/rabbitmq.asciidoc index 9a30a23f133..4c07b4150ff 100644 --- a/metricbeat/docs/modules/rabbitmq.asciidoc +++ b/metricbeat/docs/modules/rabbitmq.asciidoc @@ -19,7 +19,7 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: rabbitmq - metricsets: ["node"] + metricsets: ["node", "queue"] period: 10s hosts: ["localhost:15672"] @@ -34,5 +34,9 @@ The following metricsets are available: * <> +* <> + include::rabbitmq/node.asciidoc[] +include::rabbitmq/queue.asciidoc[] + diff --git a/metricbeat/docs/modules/rabbitmq/queue.asciidoc b/metricbeat/docs/modules/rabbitmq/queue.asciidoc new file mode 100644 index 00000000000..acb8ea89dd6 --- /dev/null +++ b/metricbeat/docs/modules/rabbitmq/queue.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-rabbitmq-queue]] +include::../../../module/rabbitmq/queue/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/rabbitmq/queue/_meta/data.json[] +---- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index dc9cff1a8a5..b06d4359baf 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -87,6 +87,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/prometheus/stats" _ "github.com/elastic/beats/metricbeat/module/rabbitmq" _ "github.com/elastic/beats/metricbeat/module/rabbitmq/node" + _ "github.com/elastic/beats/metricbeat/module/rabbitmq/queue" _ "github.com/elastic/beats/metricbeat/module/redis" _ "github.com/elastic/beats/metricbeat/module/redis/info" _ "github.com/elastic/beats/metricbeat/module/redis/keyspace" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 5dba70d55d8..081934a61a1 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -378,7 +378,7 @@ metricbeat.modules: #------------------------------ RabbitMQ Module ------------------------------ - module: rabbitmq - metricsets: ["node"] + metricsets: ["node", "queue"] period: 10s hosts: ["localhost:15672"] diff --git a/metricbeat/module/rabbitmq/_meta/config.yml b/metricbeat/module/rabbitmq/_meta/config.yml index f45427f547b..c0343876425 100644 --- a/metricbeat/module/rabbitmq/_meta/config.yml +++ b/metricbeat/module/rabbitmq/_meta/config.yml @@ -1,5 +1,5 @@ - module: rabbitmq - metricsets: ["node"] + metricsets: ["node", "queue"] period: 10s hosts: ["localhost:15672"] diff --git a/metricbeat/module/rabbitmq/_meta/testdata/queue_sample_response.json b/metricbeat/module/rabbitmq/_meta/testdata/queue_sample_response.json new file mode 100644 index 00000000000..15eb21fe863 --- /dev/null +++ b/metricbeat/module/rabbitmq/_meta/testdata/queue_sample_response.json @@ -0,0 +1,145 @@ +[ + { + "memory": 232720, + "message_stats": { + "disk_reads": 212, + "disk_reads_details": { + "rate": 0 + }, + "disk_writes": 121, + "disk_writes_details": { + "rate": 0 + }, + "deliver": 15, + "deliver_details": { + "rate": 0 + }, + "deliver_no_ack": 0, + "deliver_no_ack_details": { + "rate": 0 + }, + "get": 0, + "get_details": { + "rate": 0 + }, + "get_no_ack": 38, + "get_no_ack_details": { + "rate": 0 + }, + "publish": 121, + "publish_details": { + "rate": 0 + }, + "publish_in": 0, + "publish_in_details": { + "rate": 0 + }, + "publish_out": 0, + "publish_out_details": { + "rate": 0 + }, + "ack": 9, + "ack_details": { + "rate": 0 + }, + "deliver_get": 53, + "deliver_get_details": { + "rate": 0 + }, + "confirm": 0, + "confirm_details": { + "rate": 0 + }, + "return_unroutable": 0, + "return_unroutable_details": { + "rate": 0 + }, + "redeliver": 3, + "redeliver_details": { + "rate": 0 + } + }, + "reductions": 787128, + "reductions_details": { + "rate": 0 + }, + "messages": 74, + "messages_details": { + "rate": 0 + }, + "messages_ready": 71, + "messages_ready_details": { + "rate": 0 + }, + "messages_unacknowledged": 3, + "messages_unacknowledged_details": { + "rate": 0 + }, + "idle_since": "2017-07-28 23:45:52", + "consumer_utilisation": 0.7, + "policy": null, + "exclusive_consumer_tag": null, + "consumers": 3, + "recoverable_slaves": null, + "state": "running", + "garbage_collection": { + "min_bin_vheap_size": 46422, + "min_heap_size": 233, + "fullsweep_after": 65535, + "minor_gcs": 0 + }, + "messages_ram": 74, + "messages_ready_ram": 71, + "messages_unacknowledged_ram": 3, + "messages_persistent": 73, + "message_bytes": 101824, + "message_bytes_ready": 97696, + "message_bytes_unacknowledged": 4128, + "message_bytes_ram": 101824, + "message_bytes_persistent": 101824, + "head_message_timestamp": 1501250275, + "disk_reads": 212, + "disk_writes": 121, + "backing_queue_status": { + "priority_lengths": { + "0": 0, + "1": 71, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "mode": "default", + "q1": 0, + "q2": 0, + "delta": [ + "delta", + "todo", + "todo", + "todo" + ], + "q3": 0, + "q4": 71, + "len": 71, + "target_ram_count": "infinity", + "next_seq_id": 121, + "avg_ingress_rate": 0, + "avg_egress_rate": 0.00019793395296866087, + "avg_ack_ingress_rate": 0.00019793395296866087, + "avg_ack_egress_rate": 0.00019793395296866087 + }, + "node": "rabbit@localhost", + "arguments": { + "x-max-priority": 9 + }, + "exclusive": false, + "auto_delete": false, + "durable": true, + "vhost": "/", + "name": "queuenamehere" + } +] \ No newline at end of file diff --git a/metricbeat/module/rabbitmq/node/_meta/data.json b/metricbeat/module/rabbitmq/node/_meta/data.json index 24702fec3a2..afd97479cf6 100644 --- a/metricbeat/module/rabbitmq/node/_meta/data.json +++ b/metricbeat/module/rabbitmq/node/_meta/data.json @@ -1,19 +1,123 @@ { - "@timestamp":"2016-05-23T08:05:34.853Z", - "beat":{ - "hostname":"beathost", - "name":"beathost" + "@timestamp": "2017-08-21T21:06:45.009Z", + "@metadata": { + "beat": "metricbeat", + "type": "doc" }, - "metricset":{ - "host":"localhost", - "module":"rabbitmq", - "name":"node", - "rtt":44269 - }, - "rabbitmq":{ - "node":{ - "example": "node" + "rabbitmq": { + "node": { + "queue": { + "index": { + "journal_write": { + "count": 676 + }, + "read": {}, + "write": { + "count": 92 + } + } + }, + "processors": 8, + "msg": { + "store_read": {}, + "store_write": {} + }, + "run": { + "queue": 0 + }, + "type": "disc", + "disk": { + "free": { + "bytes": 17315815424, + "limit": { + "bytes": 50000000 + } + } + }, + "mnesia": { + "ram": { + "tx": { + "count": 87 + } + }, + "disk": { + "tx": { + "count": 740 + } + } + }, + "socket": { + "used": 0, + "total": 138 + }, + "uptime": 888613658, + "io": { + "file_handle": { + "open_attempt": { + "avg": {} + } + }, + "read": { + "count": 1, + "avg": { + "ms": 0 + }, + "bytes": 1 + }, + "reopen": { + "count": 1 + }, + "seek": { + "count": 450, + "avg": { + "ms": 7 + } + }, + "sync": { + "count": 531, + "avg": { + "ms": 0 + } + }, + "write": { + "avg": { + "ms": 0 + }, + "bytes": 346498, + "count": 531 + } + }, + "fd": { + "total": 256 + }, + "gc": { + "reclaimed": {}, + "num": {} + }, + "name": "rabbit@localhost", + "proc": { + "total": 1048576, + "used": 252 + }, + "mem": { + "limit": { + "bytes": 6309860147 + }, + "used": { + "bytes": 62095872 + } } + } + }, + "metricset": { + "module": "rabbitmq", + "name": "node", + "host": "localhost:15672", + "rtt": 30110 }, - "type":"metricsets" -} + "beat": { + "name": "name", + "hostname": "hostname", + "version": "7.0.0-alpha1" + } + } \ No newline at end of file diff --git a/metricbeat/module/rabbitmq/queue/_meta/data.json b/metricbeat/module/rabbitmq/queue/_meta/data.json new file mode 100644 index 00000000000..ca357e34c24 --- /dev/null +++ b/metricbeat/module/rabbitmq/queue/_meta/data.json @@ -0,0 +1,61 @@ +{ + "@timestamp": "2017-08-21T21:06:47.437Z", + "@metadata": { + "beat": "metricbeat", + "type": "doc" + }, + "rabbitmq": { + "queue": { + "disk": { + "writes": { + "count": 0 + }, + "reads": { + "count": 0 + } + }, + "consumers": { + "count": 5, + "utilisation": {} + }, + "auto_delete": false, + "memory": { + "bytes": 144392 + }, + "state": "running", + "messages": { + "total": { + "count": 0 + }, + "ready": { + "count": 0 + }, + "unacknowledged": { + "count": 0 + }, + "persistent": { + "count": 0 + } + }, + "durable": true, + "vhost": "/", + "arguments": { + "max_priority": 9 + }, + "exclusive": false, + "node": "rabbit@localhost", + "name": "myfancyqueue" + } + }, + "metricset": { + "module": "rabbitmq", + "name": "queue", + "host": "localhost:15672", + "rtt": 6730 + }, + "beat": { + "version": "7.0.0-alpha1", + "name": "name", + "hostname": "hostname" + } + } \ No newline at end of file diff --git a/metricbeat/module/rabbitmq/queue/_meta/docs.asciidoc b/metricbeat/module/rabbitmq/queue/_meta/docs.asciidoc new file mode 100644 index 00000000000..ef655648f98 --- /dev/null +++ b/metricbeat/module/rabbitmq/queue/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== rabbitmq queue MetricSet + +This is the queue metricset of the module rabbitmq. diff --git a/metricbeat/module/rabbitmq/queue/_meta/fields.yml b/metricbeat/module/rabbitmq/queue/_meta/fields.yml new file mode 100644 index 00000000000..0fbe15dbab9 --- /dev/null +++ b/metricbeat/module/rabbitmq/queue/_meta/fields.yml @@ -0,0 +1,75 @@ +- name: queue + type: group + description: > + queue + fields: + - name: name + type: keyword + description: > + The name of the queue with non-ASCII characters escaped as in C. + - name: vhost + type: keyword + description: > + Virtual host name with non-ASCII characters escaped as in C. + - name: durable + type: boolean + description: > + Whether or not the queue survives server restarts. + - name: auto_delete + type: boolean + description: > + Whether the queue will be deleted automatically when no longer used. + - name: exclusive + type: boolean + description: > + Whether the queue is exclusive (i.e. has owner_pid). + - name: node + type: keyword + description: > + Node name. + - name: state + type: keyword + description: > + The state of the queue. Normally 'running', but may be "{syncing, MsgCount}" if the queue is synchronising. Queues which are located on cluster nodes that are currently down will be shown with a status of 'down'. + - name: arguments.max_priority + type: long + description: > + Maximum number of priority levels for the queue to support. + - name: consumers.count + type: long + description: > + Number of consumers. + - name: consumers.utilisation.pct + type: long + format: percentage + description: > + Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count. + - name: messages.total.count + type: long + description: > + Sum of ready and unacknowledged messages (queue depth). + - name: messages.ready.count + type: long + description: > + Number of messages ready to be delivered to clients. + - name: messages.unacknowledged.count + type: long + description: > + Number of messages delivered to clients but not yet acknowledged. + - name: messages.persistent.count + type: long + description: > + Total number of persistent messages in the queue (will always be 0 for transient queues). + - name: memory.bytes + type: long + format: bytes + description: > + Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures. + - name: disk.reads.count + type: long + description: > + Total number of times messages have been read from disk by this queue since it started. + - name: disk.writes.count + type: long + description: > + Total number of times messages have been written to disk by this queue since it started. diff --git a/metricbeat/module/rabbitmq/queue/data.go b/metricbeat/module/rabbitmq/queue/data.go new file mode 100644 index 00000000000..6c7a04b7f74 --- /dev/null +++ b/metricbeat/module/rabbitmq/queue/data.go @@ -0,0 +1,79 @@ +package queue + +import ( + "encoding/json" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/libbeat/logp" +) + +var ( + schema = s.Schema{ + "name": c.Str("name"), + "vhost": c.Str("vhost"), + "durable": c.Bool("durable"), + "auto_delete": c.Bool("auto_delete"), + "exclusive": c.Bool("exclusive"), + "node": c.Str("node"), + "state": c.Str("state"), + "arguments": c.Dict("arguments", s.Schema{ + "max_priority": c.Int("x-max-priority", s.Optional), + }), + "consumers": s.Object{ + "count": c.Int("consumers"), + "utilisation": s.Object{ + "pct": c.Int("consumer_utilisation", s.Optional), + }, + }, + "messages": s.Object{ + "total": s.Object{ + "count": c.Int("messages"), + }, + "ready": s.Object{ + "count": c.Int("messages_ready"), + }, + "unacknowledged": s.Object{ + "count": c.Int("messages_unacknowledged"), + }, + "persistent": s.Object{ + "count": c.Int("messages_persistent"), + }, + }, + "memory": s.Object{ + "bytes": c.Int("memory"), + }, + "disk": s.Object{ + "reads": s.Object{ + "count": c.Int("disk_reads"), + }, + "writes": s.Object{ + "count": c.Int("disk_writes"), + }, + }, + } +) + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var queues []map[string]interface{} + err := json.Unmarshal(content, &queues) + if err != nil { + logp.Err("Error: ", err) + } + + events := []common.MapStr{} + errors := s.NewErrors() + + for _, queue := range queues { + event, errs := eventMapping(queue) + events = append(events, event) + errors.AddErrors(errs) + } + + return events, errors +} + +func eventMapping(queue map[string]interface{}) (common.MapStr, *s.Errors) { + return schema.Apply(queue) +} diff --git a/metricbeat/module/rabbitmq/queue/node_integration_test.go b/metricbeat/module/rabbitmq/queue/node_integration_test.go new file mode 100644 index 00000000000..7801aa7524e --- /dev/null +++ b/metricbeat/module/rabbitmq/queue/node_integration_test.go @@ -0,0 +1,60 @@ +package queue + +import ( + "fmt" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + err := mbtest.WriteEvents(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "rabbitmq", + "metricsets": []string{"queue"}, + "hosts": getTestRabbitMQHost(), + "username": getTestRabbitMQUsername(), + "password": getTestRabbitMQPassword(), + } +} + +const ( + rabbitmqDefaultHost = "localhost" + rabbitmqDefaultPort = "15672" + rabbitmqDefaultUsername = "guest" + rabbitmqDefaultPassword = "guest" +) + +func getTestRabbitMQHost() string { + return fmt.Sprintf("%v:%v", + getenv("RABBITMQ_HOST", rabbitmqDefaultHost), + getenv("RABBITMQ_PORT", rabbitmqDefaultPort), + ) +} + +func getTestRabbitMQUsername() string { + return getenv("RABBITMQ_USERNAME", rabbitmqDefaultUsername) +} + +func getTestRabbitMQPassword() string { + return getenv("RABBITMQ_PASSWORD", rabbitmqDefaultPassword) +} + +func getenv(name, defaultValue string) string { + return strDefault(os.Getenv(name), defaultValue) +} + +func strDefault(a, defaults string) string { + if len(a) == 0 { + return defaults + } + return a +} diff --git a/metricbeat/module/rabbitmq/queue/node_test.go b/metricbeat/module/rabbitmq/queue/node_test.go new file mode 100644 index 00000000000..24cfbea8f1f --- /dev/null +++ b/metricbeat/module/rabbitmq/queue/node_test.go @@ -0,0 +1,76 @@ +package queue + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + + response, err := ioutil.ReadFile(absPath + "/queue_sample_response.json") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "rabbitmq", + "metricsets": []string{"queue"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewEventsFetcher(t, config) + events, err := f.Fetch() + event := events[0] + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + assert.EqualValues(t, "queuenamehere", event["name"]) + assert.EqualValues(t, "/", event["vhost"]) + assert.EqualValues(t, true, event["durable"]) + assert.EqualValues(t, false, event["auto_delete"]) + assert.EqualValues(t, false, event["exclusive"]) + assert.EqualValues(t, "running", event["state"]) + assert.EqualValues(t, "rabbit@localhost", event["node"]) + + arguments := event["arguments"].(common.MapStr) + assert.EqualValues(t, 9, arguments["max_priority"]) + + consumers := event["consumers"].(common.MapStr) + utilisation := consumers["utilisation"].(common.MapStr) + assert.EqualValues(t, 3, consumers["count"]) + assert.EqualValues(t, 0.7, utilisation["pct"]) + + memory := event["memory"].(common.MapStr) + assert.EqualValues(t, 232720, memory["bytes"]) + + messages := event["messages"].(common.MapStr) + total := messages["total"].(common.MapStr) + ready := messages["ready"].(common.MapStr) + unacknowledged := messages["unacknowledged"].(common.MapStr) + persistent := messages["persistent"].(common.MapStr) + assert.EqualValues(t, 74, total["count"]) + assert.EqualValues(t, 71, ready["count"]) + assert.EqualValues(t, 3, unacknowledged["count"]) + assert.EqualValues(t, 73, persistent["count"]) + + disk := event["disk"].(common.MapStr) + reads := disk["reads"].(common.MapStr) + writes := disk["writes"].(common.MapStr) + assert.EqualValues(t, 212, reads["count"]) + assert.EqualValues(t, 121, writes["count"]) +} diff --git a/metricbeat/module/rabbitmq/queue/queue.go b/metricbeat/module/rabbitmq/queue/queue.go new file mode 100644 index 00000000000..688361fe713 --- /dev/null +++ b/metricbeat/module/rabbitmq/queue/queue.go @@ -0,0 +1,55 @@ +package queue + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/api/queues" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +func init() { + if err := mb.Registry.AddMetricSet("rabbitmq", "queue", New, hostParser); err != nil { + panic(err) + } +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The rabbitmq queue metricset is experimental") + + http := helper.NewHTTP(base) + http.SetHeader("Accept", "application/json") + + return &MetricSet{ + base, + http, + }, nil +} + +func (m *MetricSet) Fetch() ([]common.MapStr, error) { + content, err := m.HTTP.FetchContent() + + if err != nil { + return nil, err + } + + events, _ := eventsMapping(content) + return events, nil +} diff --git a/metricbeat/modules.d/rabbitmq.yml.disabled b/metricbeat/modules.d/rabbitmq.yml.disabled index f45427f547b..c0343876425 100644 --- a/metricbeat/modules.d/rabbitmq.yml.disabled +++ b/metricbeat/modules.d/rabbitmq.yml.disabled @@ -1,5 +1,5 @@ - module: rabbitmq - metricsets: ["node"] + metricsets: ["node", "queue"] period: 10s hosts: ["localhost:15672"] From e0d7c16f2d8697c29d2b7e579e891a52b385165e Mon Sep 17 00:00:00 2001 From: Monica Sarbu Date: Wed, 27 Sep 2017 16:10:38 +0200 Subject: [PATCH 53/96] Create a separate deashboard for Nginx access and error logs (#5240) --- .../dashboard/Filebeat-nginx-logs.json | 99 ++++++++++++++++ .../dashboard/Filebeat-nginx-overview.json | 112 ++++++++++-------- filebeat/module/nginx/module.yml | 5 +- 3 files changed, 168 insertions(+), 48 deletions(-) create mode 100644 filebeat/module/nginx/_meta/kibana/default/dashboard/Filebeat-nginx-logs.json diff --git a/filebeat/module/nginx/_meta/kibana/default/dashboard/Filebeat-nginx-logs.json b/filebeat/module/nginx/_meta/kibana/default/dashboard/Filebeat-nginx-logs.json new file mode 100644 index 00000000000..788d962331f --- /dev/null +++ b/filebeat/module/nginx/_meta/kibana/default/dashboard/Filebeat-nginx-logs.json @@ -0,0 +1,99 @@ +{ + "objects": [ + { + "attributes": { + "columns": [ + "nginx.error.level", + "nginx.error.message" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"language\":\"lucene\",\"query\":\"_exists_:nginx AND _exists_:nginx.error.message\"},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"highlightAll\":true,\"version\":true}" + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Nginx error logs [Filebeat Nginx]", + "version": 1 + }, + "id": "9eb25600-a1f0-11e7-928f-5dbe6f6f5519", + "type": "search", + "version": 1 + }, + { + "attributes": { + "columns": [ + "nginx.access.url", + "nginx.access.method", + "nginx.access.response_code", + "nginx.access.body_sent.bytes" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"language\":\"lucene\",\"query\":\"_exists_:nginx AND _exists_:nginx.access.url\"},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"highlightAll\":true,\"version\":true}" + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Nginx access logs [Filebeat Nginx]", + "version": 1 + }, + "id": "6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519", + "type": "search", + "version": 4 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "title": "Access logs over time [Filebeat Nginx]", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"title\":\"Access logs over time [Filebeat Nginx]\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"nginx.access.url\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"split_filters\":[{\"color\":\"#68BC00\",\"id\":\"1db649a0-a1f3-11e7-a062-a1c3587f4874\"}],\"label\":\"Access logs\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"show_grid\":1,\"background_color_rules\":[{\"id\":\"3189aa80-a1f3-11e7-a062-a1c3587f4874\"}],\"annotations\":[{\"id\":\"970b1420-a1f3-11e7-a062-a1c3587f4874\",\"color\":\"#F00\",\"index_pattern\":\"*\",\"time_field\":\"@timestamp\",\"icon\":\"fa-tag\",\"ignore_global_filters\":1,\"ignore_panel_filters\":1}],\"filter\":\"fileset.module:nginx AND fileset.name:access\",\"legend_position\":\"bottom\"},\"aggs\":[]}" + }, + "id": "1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519", + "type": "visualization", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "title": "Dashboards [Filebeat Nginx]", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"title\":\"Dashboards [Filebeat Nginx]\",\"type\":\"markdown\",\"params\":{\"fontSize\":12,\"markdown\":\"[Nginx logs overview](#/dashboard/55a9e6e0-a29e-11e7-928f-5dbe6f6f5519) | [Nginx access and error logs](#/dashboard/046212a0-a2a1-11e7-928f-5dbe6f6f5519)\"},\"aggs\":[]}" + }, + "id": "97109780-a2a5-11e7-928f-5dbe6f6f5519", + "type": "visualization", + "version": 1 + }, + { + "attributes": { + "description": "Dashboard for the Filebeat Nginx module", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"highlightAll\":true,\"version\":true}" + }, + "optionsJSON": "{\"darkTheme\":false}", + "panelsJSON": "[{\"col\":1,\"columns\":[\"nginx.error.level\",\"nginx.error.message\"],\"id\":\"9eb25600-a1f0-11e7-928f-5dbe6f6f5519\",\"panelIndex\":11,\"row\":5,\"size_x\":12,\"size_y\":3,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"columns\":[\"nginx.access.url\",\"nginx.access.method\",\"nginx.access.response_code\",\"nginx.access.body_sent.bytes\"],\"id\":\"6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519\",\"panelIndex\":16,\"row\":8,\"size_x\":12,\"size_y\":7,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519\",\"panelIndex\":18,\"row\":2,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":1,\"panelIndex\":19,\"type\":\"visualization\",\"id\":\"97109780-a2a5-11e7-928f-5dbe6f6f5519\",\"col\":1,\"row\":1}]", + "timeRestore": false, + "title": "[Filebeat Nginx] Access and error logs", + "uiStateJSON": "{}", + "version": 1 + }, + "id": "046212a0-a2a1-11e7-928f-5dbe6f6f5519", + "type": "dashboard", + "version": 2 + } + ], + "version": "6.0.0-beta2" +} \ No newline at end of file diff --git a/filebeat/module/nginx/_meta/kibana/default/dashboard/Filebeat-nginx-overview.json b/filebeat/module/nginx/_meta/kibana/default/dashboard/Filebeat-nginx-overview.json index da29df38743..0bfd3ff47bc 100644 --- a/filebeat/module/nginx/_meta/kibana/default/dashboard/Filebeat-nginx-overview.json +++ b/filebeat/module/nginx/_meta/kibana/default/dashboard/Filebeat-nginx-overview.json @@ -6,14 +6,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": "{\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" }, - "title": "Errors over time [Filebeat Nginx]", + "title": "Browsers breakdown [Filebeat Nginx]", "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"title\": \"Errors over time\",\n \"type\": \"area\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"smoothLines\": false,\n \"scale\": \"linear\",\n \"interpolate\": \"linear\",\n \"mode\": \"stacked\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"nginx.error.level\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}" + "visState": "{\n \"title\": \"Nginx Access Browsers\",\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"bottom\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.user_agent.name\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.user_agent.major\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}" }, - "id": "Errors-over-time", + "id": "Nginx-Access-Browsers", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -21,101 +21,119 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": "{\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" }, - "title": "Browsers breakdown [Filebeat Nginx]", + "title": "Operating systems breakdown [Filebeat Nginx]", "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"title\": \"Nginx Access Browsers\",\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"bottom\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.user_agent.name\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.user_agent.major\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}" + "visState": "{\n \"title\": \"Nginx Access OSes\",\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"bottom\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.user_agent.os_name\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.user_agent.os_major\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}" }, - "id": "Nginx-Access-Browsers", + "id": "Nginx-Access-OSes", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" + "searchSourceJSON": "{\n \"filter\": []\n}" }, - "title": "Operating systems breakdown [Filebeat Nginx]", - "uiStateJSON": "{}", + "savedSearchId": "Filebeat-Nginx-module", + "title": "Access Map [Filebeat Nginx]", + "uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}", "version": 1, - "visState": "{\n \"title\": \"Nginx Access OSes\",\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"bottom\",\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.user_agent.os_name\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.user_agent.os_major\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}" + "visState": "{\n \"aggs\": [\n {\n \"enabled\": true,\n \"id\": \"1\",\n \"params\": {},\n \"schema\": \"metric\",\n \"type\": \"count\"\n },\n {\n \"enabled\": true,\n \"id\": \"2\",\n \"params\": {\n \"autoPrecision\": true,\n \"field\": \"nginx.access.geoip.location\"\n },\n \"schema\": \"segment\",\n \"type\": \"geohash_grid\"\n }\n ],\n \"listeners\": {},\n \"params\": {\n \"addTooltip\": true,\n \"heatBlur\": 15,\n \"heatMaxZoom\": 16,\n \"heatMinOpacity\": 0.1,\n \"heatNormalizeData\": true,\n \"heatRadius\": 25,\n \"isDesaturated\": true,\n \"legendPosition\": \"bottomright\",\n \"mapCenter\": [\n 15,\n 5\n ],\n \"mapType\": \"Scaled Circle Markers\",\n \"mapZoom\": 2,\n \"wms\": {\n \"enabled\": false,\n \"options\": {\n \"attribution\": \"Maps provided by USGS\",\n \"format\": \"image/png\",\n \"layers\": \"0\",\n \"styles\": \"\",\n \"transparent\": true,\n \"version\": \"1.3.0\"\n },\n \"url\": \"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"\n }\n },\n \"title\": \"Nginx Access Map\",\n \"type\": \"tile_map\"\n}" }, - "id": "Nginx-Access-OSes", + "id": "Nginx-Access-Map", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"filter\": []\n}" + "searchSourceJSON": "{}" }, - "savedSearchId": "Filebeat-Nginx-module", "title": "Response codes over time [Filebeat Nginx]", - "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}", + "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"title\": \"New Visualization\",\n \"type\": \"histogram\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"scale\": \"linear\",\n \"mode\": \"stacked\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"nginx.access.response_code\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}" + "visState": "{\"title\":\"Response codes over time [Filebeat Nginx]\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"filters\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"terms_field\":\"nginx.access.response_code\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"label\":\"\",\"split_filters\":[{\"filter\":\"nginx.access.response_code:[200 TO 299]\",\"label\":\"200s\",\"color\":\"#68BC00\",\"id\":\"5acdc750-a29d-11e7-a062-a1c3587f4874\"},{\"filter\":\"nginx.access.response_code:[300 TO 399]\",\"label\":\"300s\",\"color\":\"rgba(252,196,0,1)\",\"id\":\"6efd2ae0-a29d-11e7-a062-a1c3587f4874\"},{\"filter\":\"nginx.access.response_code:[400 TO 499]\",\"label\":\"400s\",\"color\":\"rgba(211,49,21,1)\",\"id\":\"76089a90-a29d-11e7-a062-a1c3587f4874\"},{\"filter\":\"nginx.access.response_code:[500 TO 599]\",\"label\":\"500s\",\"color\":\"rgba(171,20,158,1)\",\"id\":\"7c7929d0-a29d-11e7-a062-a1c3587f4874\"}]}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"show_grid\":1,\"legend_position\":\"bottom\",\"filter\":\"fileset.module:nginx AND fileset.name:access\"},\"aggs\":[]}" }, - "id": "New-Visualization", + "id": "b70b1b20-a1f4-11e7-928f-5dbe6f6f5519", "type": "visualization", - "version": 2 + "version": 7 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}" + "searchSourceJSON": "{}" }, - "title": "Response codes by top URLs [Filebeat Nginx]", - "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#629E51\",\n \"404\": \"#0A50A1\"\n }\n }\n}", + "title": "Top pages [Filebeat Nginx]", + "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"title\": \"Nginx Access Response codes by top URLs\",\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"isDonut\": false\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"split\",\n \"params\": {\n \"field\": \"nginx.access.url\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\",\n \"row\": false\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"nginx.access.response_code\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}" + "visState": "{\"title\":\"Top pages [Filebeat Nginx]\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"top_n\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"nginx.access.url\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"value_template\":\"\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"show_grid\":1,\"bar_color_rules\":[{\"id\":\"6252c320-a1f5-11e7-92ba-5d0b8663aece\"}],\"filter\":\"fileset.module:nginx AND fileset.name:access\"},\"aggs\":[]}" }, - "id": "Nginx-Access-Response-codes-by-top-URLs", + "id": "9184fa00-a1f5-11e7-928f-5dbe6f6f5519", "type": "visualization", - "version": 2 + "version": 3 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"filter\": [],\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"_exists_:nginx.access\",\n \"analyze_wildcard\": true\n }\n },\n \"highlight\": {\n \"pre_tags\": [\n \"@kibana-highlighted-field@\"\n ],\n \"post_tags\": [\n \"@/kibana-highlighted-field@\"\n ],\n \"fields\": {\n \"*\": {}\n },\n \"require_field_match\": false,\n \"fragment_size\": 2147483647\n }\n}" + "searchSourceJSON": "{}" }, - "title": "Sent Byte Size [Filebeat Nginx]", + "title": "Errors over time [Filebeat Nginx]", "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"title\": \"Sent sizes\",\n \"type\": \"line\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"showCircles\": true,\n \"smoothLines\": true,\n \"interpolate\": \"linear\",\n \"scale\": \"linear\",\n \"drawLinesBetweenPoints\": true,\n \"radiusRatio\": \"17\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"sum\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"nginx.access.body_sent.bytes\",\n \"customLabel\": \"Data sent\"\n }\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"radius\",\n \"params\": {}\n }\n ],\n \"listeners\": {}\n}" + "visState": "{\"title\":\"Errors over time [Filebeat Nginx]\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"rgba(211,49,21,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"nginx.error.level\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"show_grid\":1,\"filter\":\"fileset.module:nginx AND fileset.name:error\",\"legend_position\":\"bottom\"},\"aggs\":[]}" }, - "id": "Sent-sizes", + "id": "46322e50-a1f6-11e7-928f-5dbe6f6f5519", "type": "visualization", - "version": 2 + "version": 5 }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"filter\": []\n}" + "searchSourceJSON": "{}" }, - "savedSearchId": "Filebeat-Nginx-module", - "title": "Access Map [Filebeat Nginx]", - "uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}", + "title": "Data Volume [Filebeat Nginx]", + "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"aggs\": [\n {\n \"enabled\": true,\n \"id\": \"1\",\n \"params\": {},\n \"schema\": \"metric\",\n \"type\": \"count\"\n },\n {\n \"enabled\": true,\n \"id\": \"2\",\n \"params\": {\n \"autoPrecision\": true,\n \"field\": \"nginx.access.geoip.location\"\n },\n \"schema\": \"segment\",\n \"type\": \"geohash_grid\"\n }\n ],\n \"listeners\": {},\n \"params\": {\n \"addTooltip\": true,\n \"heatBlur\": 15,\n \"heatMaxZoom\": 16,\n \"heatMinOpacity\": 0.1,\n \"heatNormalizeData\": true,\n \"heatRadius\": 25,\n \"isDesaturated\": true,\n \"legendPosition\": \"bottomright\",\n \"mapCenter\": [\n 15,\n 5\n ],\n \"mapType\": \"Scaled Circle Markers\",\n \"mapZoom\": 2,\n \"wms\": {\n \"enabled\": false,\n \"options\": {\n \"attribution\": \"Maps provided by USGS\",\n \"format\": \"image/png\",\n \"layers\": \"0\",\n \"styles\": \"\",\n \"transparent\": true,\n \"version\": \"1.3.0\"\n },\n \"url\": \"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"\n }\n },\n \"title\": \"Nginx Access Map\",\n \"type\": \"tile_map\"\n}" + "visState": "{\"title\":\"Data Volume [Filebeat Nginx]\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"nginx.access.body_sent.bytes\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"split_filters\":[{\"filter\":\"nginx.access.response_code:[200 TO 299]\",\"label\":\"200s\",\"color\":\"#68BC00\",\"id\":\"7c343c20-a29e-11e7-a062-a1c3587f4874\"}],\"label\":\"\",\"terms_field\":null}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"show_grid\":1,\"filter\":\"fileset.module: nginx AND fileset.name: access\",\"legend_position\":\"bottom\"},\"aggs\":[]}" }, - "id": "Nginx-Access-Map", + "id": "0dd6f320-a29f-11e7-928f-5dbe6f6f5519", "type": "visualization", "version": 2 }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "title": "Dashboards [Filebeat Nginx]", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"title\":\"Dashboards [Filebeat Nginx]\",\"type\":\"markdown\",\"params\":{\"fontSize\":12,\"markdown\":\"[Nginx logs overview](#/dashboard/55a9e6e0-a29e-11e7-928f-5dbe6f6f5519) | [Nginx access and error logs](#/dashboard/046212a0-a2a1-11e7-928f-5dbe6f6f5519)\"},\"aggs\":[]}" + }, + "id": "97109780-a2a5-11e7-928f-5dbe6f6f5519", + "type": "visualization", + "version": 1 + }, { "attributes": { "columns": [ - "_source" + "nginx.access.url", + "nginx.access.method", + "nginx.access.response_code", + "nginx.access.referrer", + "nginx.access.body_sent.bytes" ], "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"index\": \"filebeat-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"_exists_:nginx\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": [],\n \"highlight\": {\n \"pre_tags\": [\n \"@kibana-highlighted-field@\"\n ],\n \"post_tags\": [\n \"@/kibana-highlighted-field@\"\n ],\n \"fields\": {\n \"*\": {}\n },\n \"require_field_match\": false,\n \"fragment_size\": 2147483647\n }\n}" + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"_exists_:nginx\"}}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"highlightAll\":true,\"version\":true}" }, "sort": [ "@timestamp", @@ -133,19 +151,19 @@ "description": "Dashboard for the Filebeat Nginx module", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"filter\": [\n {\n \"query\": {\n \"query_string\": {\n \"analyze_wildcard\": true,\n \"query\": \"*\"\n }\n }\n }\n ]\n}" + "searchSourceJSON": "{\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"highlightAll\":true,\"version\":true}" }, - "optionsJSON": "{\n \"darkTheme\": false\n}", - "panelsJSON": "[\n {\n \"col\": 9,\n \"id\": \"Errors-over-time\",\n \"panelIndex\": 2,\n \"row\": 4,\n \"size_x\": 4,\n \"size_y\": 3,\n \"type\": \"visualization\"\n },\n {\n \"col\": 1,\n \"id\": \"Nginx-Access-Browsers\",\n \"panelIndex\": 3,\n \"row\": 10,\n \"size_x\": 4,\n \"size_y\": 4,\n \"type\": \"visualization\"\n },\n {\n \"col\": 5,\n \"id\": \"Nginx-Access-OSes\",\n \"panelIndex\": 4,\n \"row\": 10,\n \"size_x\": 4,\n \"size_y\": 4,\n \"type\": \"visualization\"\n },\n {\n \"col\": 1,\n \"id\": \"New-Visualization\",\n \"panelIndex\": 5,\n \"row\": 4,\n \"size_x\": 8,\n \"size_y\": 3,\n \"type\": \"visualization\"\n },\n {\n \"col\": 1,\n \"id\": \"Nginx-Access-Response-codes-by-top-URLs\",\n \"panelIndex\": 6,\n \"row\": 7,\n \"size_x\": 12,\n \"size_y\": 3,\n \"type\": \"visualization\"\n },\n {\n \"col\": 9,\n \"id\": \"Sent-sizes\",\n \"panelIndex\": 7,\n \"row\": 10,\n \"size_x\": 4,\n \"size_y\": 4,\n \"type\": \"visualization\"\n },\n {\n \"id\": \"Nginx-Access-Map\",\n \"type\": \"visualization\",\n \"panelIndex\": 8,\n \"size_x\": 12,\n \"size_y\": 3,\n \"col\": 1,\n \"row\": 1\n }\n]", + "optionsJSON": "{\"darkTheme\":false}", + "panelsJSON": "[{\"col\":10,\"id\":\"Nginx-Access-Browsers\",\"panelIndex\":3,\"row\":12,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Nginx-Access-OSes\",\"panelIndex\":4,\"row\":12,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Nginx-Access-Map\",\"panelIndex\":8,\"row\":2,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"b70b1b20-a1f4-11e7-928f-5dbe6f6f5519\",\"panelIndex\":13,\"row\":6,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"9184fa00-a1f5-11e7-928f-5dbe6f6f5519\",\"panelIndex\":14,\"row\":9,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"46322e50-a1f6-11e7-928f-5dbe6f6f5519\",\"panelIndex\":15,\"row\":9,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"0dd6f320-a29f-11e7-928f-5dbe6f6f5519\",\"panelIndex\":16,\"row\":12,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":1,\"panelIndex\":17,\"type\":\"visualization\",\"id\":\"97109780-a2a5-11e7-928f-5dbe6f6f5519\",\"col\":1,\"row\":1}]", "timeRestore": false, - "title": "[Filebeat Nginx] Access and error logs", - "uiStateJSON": "{\n \"P-4\": {\n \"vis\": {\n \"legendOpen\": true\n }\n },\n \"P-8\": {\n \"mapCenter\": [\n 50.51342652633956,\n -0.17578125\n ]\n }\n}", + "title": "[Filebeat Nginx] Overview", + "uiStateJSON": "{\"P-4\":{\"vis\":{\"legendOpen\":true}},\"P-8\":{\"mapBounds\":{\"bottom_right\":{\"lat\":-7.362466865535738,\"lon\":245.39062500000003},\"top_left\":{\"lat\":77.07878389624943,\"lon\":-245.74218750000003}},\"mapCenter\":[50.51342652633956,-0.17578125],\"mapCollar\":{\"top_left\":{\"lat\":90,\"lon\":-180},\"bottom_right\":{\"lat\":-49.583095,\"lon\":180},\"zoom\":2},\"mapZoom\":2}}", "version": 1 }, - "id": "Filebeat-Nginx-Dashboard", + "id": "55a9e6e0-a29e-11e7-928f-5dbe6f6f5519", "type": "dashboard", - "version": 3 + "version": 6 } ], - "version": "6.0.0-beta1-SNAPSHOT" + "version": "6.0.0-beta2" } \ No newline at end of file diff --git a/filebeat/module/nginx/module.yml b/filebeat/module/nginx/module.yml index 3fa0c0290b6..f0447f9173e 100644 --- a/filebeat/module/nginx/module.yml +++ b/filebeat/module/nginx/module.yml @@ -1,7 +1,10 @@ dashboards: -- id: Filebeat-Nginx-Dashboard +- id: 55a9e6e0-a29e-11e7-928f-5dbe6f6f5519 file: Filebeat-nginx-overview.json +- id: 046212a0-a2a1-11e7-928f-5dbe6f6f5519 + file: Filebeat-nginx-logs.json + - id: ML-Nginx-Access-Remote-IP-Count-Explorer file: ml-nginx-access-remote-ip-count-explorer.json From 9a3ab3fbaccbe375f7e159ba7a2eb1e6596a084e Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 28 Sep 2017 10:54:08 +0200 Subject: [PATCH 54/96] Remove broken options runoptions.uid and runoptions.gid (#5261) These were meant to allow dropping privileges in Packetbeat, but they no longer worked for a good while (since Golang 1.4). As shown in #3542, a better way of running Packetbeat as a non-root user is to use capabilities: setcap cap_net_raw,cap_net_admin=eip packetbeat Closes #3542. --- CHANGELOG.asciidoc | 2 + libbeat/common/droppriv/droppriv_unix.go | 41 --------------------- libbeat/common/droppriv/droppriv_windows.go | 17 --------- packetbeat/beater/packetbeat.go | 6 --- packetbeat/config/config.go | 2 - packetbeat/docs/configuring-howto.asciidoc | 3 -- packetbeat/docs/runconfig.asciidoc | 23 ------------ 7 files changed, 2 insertions(+), 92 deletions(-) delete mode 100644 libbeat/common/droppriv/droppriv_unix.go delete mode 100644 libbeat/common/droppriv/droppriv_windows.go delete mode 100644 packetbeat/docs/runconfig.asciidoc diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0b5ec119f1a..edfce35dbb8 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -25,6 +25,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di *Packetbeat* +- Remove not-working `runoptions.uid` and `runoptions.gid` options in Packetbeat. {pull}5261[5261] + *Winlogbeat* ==== Bugfixes diff --git a/libbeat/common/droppriv/droppriv_unix.go b/libbeat/common/droppriv/droppriv_unix.go deleted file mode 100644 index d9516afd1ed..00000000000 --- a/libbeat/common/droppriv/droppriv_unix.go +++ /dev/null @@ -1,41 +0,0 @@ -// +build !windows - -package droppriv - -import ( - "errors" - "fmt" - "syscall" - - "github.com/elastic/beats/libbeat/logp" -) - -type RunOptions struct { - UID *int - GID *int -} - -func DropPrivileges(config RunOptions) error { - var err error - - if config.UID == nil { - // not found, no dropping privileges but no err - return nil - } - - if config.GID == nil { - return errors.New("GID must be specified for dropping privileges") - } - - logp.Info("Switching to user: %d.%d", *config.UID, *config.GID) - - if err = syscall.Setgid(*config.GID); err != nil { - return fmt.Errorf("setgid: %s", err.Error()) - } - - if err = syscall.Setuid(*config.UID); err != nil { - return fmt.Errorf("setuid: %s", err.Error()) - } - - return nil -} diff --git a/libbeat/common/droppriv/droppriv_windows.go b/libbeat/common/droppriv/droppriv_windows.go deleted file mode 100644 index a7c00f790ae..00000000000 --- a/libbeat/common/droppriv/droppriv_windows.go +++ /dev/null @@ -1,17 +0,0 @@ -package droppriv - -import "errors" - -type RunOptions struct { - Uid *int - Gid *int -} - -func DropPrivileges(config RunOptions) error { - if config.Uid == nil { - // not found, no dropping privileges but no err - return nil - } - - return errors.New("Dropping privileges is not supported on Windows") -} diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go index 768ca7e790a..6e0d265f391 100644 --- a/packetbeat/beater/packetbeat.go +++ b/packetbeat/beater/packetbeat.go @@ -11,7 +11,6 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/droppriv" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/service" @@ -177,11 +176,6 @@ func (pb *packetbeat) Run(b *beat.Beat) error { } }() - // This needs to be after the sniffer Init but before the sniffer Run. - if err := droppriv.DropPrivileges(pb.config.RunOptions); err != nil { - return err - } - defer pb.transPub.Stop() timeout := pb.config.ShutdownTimeout diff --git a/packetbeat/config/config.go b/packetbeat/config/config.go index d141a357a28..62a8fbe1a09 100644 --- a/packetbeat/config/config.go +++ b/packetbeat/config/config.go @@ -4,7 +4,6 @@ import ( "time" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/droppriv" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/packetbeat/procs" ) @@ -17,7 +16,6 @@ type Config struct { Procs procs.ProcsConfig `config:"procs"` IgnoreOutgoing bool `config:"ignore_outgoing"` ShutdownTimeout time.Duration `config:"shutdown_timeout"` - RunOptions droppriv.RunOptions } type InterfacesConfig struct { diff --git a/packetbeat/docs/configuring-howto.asciidoc b/packetbeat/docs/configuring-howto.asciidoc index a324997beab..bab608cf2c2 100644 --- a/packetbeat/docs/configuring-howto.asciidoc +++ b/packetbeat/docs/configuring-howto.asciidoc @@ -34,7 +34,6 @@ The following topics describe how to configure Packetbeat: * <> * <> * <> -* <> * <> * <> @@ -63,8 +62,6 @@ include::../../libbeat/docs/setup-config.asciidoc[] include::../../libbeat/docs/loggingconfig.asciidoc[] -include::./runconfig.asciidoc[] - :standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] diff --git a/packetbeat/docs/runconfig.asciidoc b/packetbeat/docs/runconfig.asciidoc deleted file mode 100644 index 9df3543ed71..00000000000 --- a/packetbeat/docs/runconfig.asciidoc +++ /dev/null @@ -1,23 +0,0 @@ -[[configuration-run-options]] -== Set run options - -Packetbeat can drop privileges after creating the sniffing socket. -Root access is required for opening the socket, but everything else requires no -privileges. Therefore, it is recommended that you have the Beat switch users after -the initialization phase. The `uid` and `gid` settings set the User Id and Group -Id under which {beatname_uc} runs. - -WARNING: On Linux, Setuid doesn't change the uid of all threads, so the Go - garbage collector will continue to run as root. Also note that process - monitoring only works when running as root. - -Example configuration for the `runoptions` section of the +{beatname_lc}.yml+ config file: - -[source,yaml] ------------------------------------------------------------------------------- -packetbeat.runoptions: - uid: 501 - gid: 501 ------------------------------------------------------------------------------- - -The `runoptions` configuration is supported on Linux only. From 3c13108a128a07b8587656e53f5f5ea911544ec1 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Fri, 29 Sep 2017 10:34:48 +0200 Subject: [PATCH 55/96] Change field type of http header from nested to object (#5258) The field type of http headers was set to nested instead of object. In metricbeat we normally do not used nested fields. Also nested fields are not compatible with the sorting on index time feature coming in 6.0. The problem with indexing the headers is that it could lead to field explosion if there are many different headers. An alternative would be to not index the headers. For now my recommendation is if someone has too many headers, filters should be used to remove most of the entries before it is sent to Elasticsearch. --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 4 ++-- metricbeat/docs/modules/http.asciidoc | 2 ++ metricbeat/module/http/_meta/docs.asciidoc | 2 ++ metricbeat/module/http/_meta/fields.yml | 4 ++-- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index edfce35dbb8..20bcd934d79 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -61,6 +61,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Fix kubernetes events module to be able to index time fields properly. {issue}5093[5093] - The MongoDB module now connects on each fetch, to avoid stopping the whole Metricbeat instance if MongoDB is not up when starting. {pull}5120[5120] - Fixed `cmd_set` and `cmd_get` being mixed in the Memcache module. {pull}5189[5189] +- Change field type of http header from nested to object {pull}5258[5258] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3a8ce9d8277..a9138878fb1 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4039,7 +4039,7 @@ HTTP request information [float] === `http.request.header` -type: nested +type: object The HTTP headers sent @@ -4070,7 +4070,7 @@ HTTP response information [float] === `http.response.header` -type: nested +type: object The HTTP headers received diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc index 46cd1f66aaa..523fcdfbdbf 100644 --- a/metricbeat/docs/modules/http.asciidoc +++ b/metricbeat/docs/modules/http.asciidoc @@ -13,6 +13,8 @@ This module is inspired by the Logstash https://www.elastic.co/guide/en/logstash This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored. +NOTE: As the HTTP mertricsets also fetch headers, this can lead to lots of fields in Elasticsearch in case there are many different headers. If this is the case for you and you don't need the headers, we recommend to use processors to filter out the header field. + [float] === Example configuration diff --git a/metricbeat/module/http/_meta/docs.asciidoc b/metricbeat/module/http/_meta/docs.asciidoc index 3fd08235c2a..10052514896 100644 --- a/metricbeat/module/http/_meta/docs.asciidoc +++ b/metricbeat/module/http/_meta/docs.asciidoc @@ -7,3 +7,5 @@ Multiple endpoints can be configured which are polled in a regular interval and This module is inspired by the Logstash https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html[http_poller] input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured output channels, e.g. Logstash or Elasticsearch. This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored. + +NOTE: As the HTTP mertricsets also fetch headers, this can lead to lots of fields in Elasticsearch in case there are many different headers. If this is the case for you and you don't need the headers, we recommend to use processors to filter out the header field. diff --git a/metricbeat/module/http/_meta/fields.yml b/metricbeat/module/http/_meta/fields.yml index d158a5e6237..5f760168563 100644 --- a/metricbeat/module/http/_meta/fields.yml +++ b/metricbeat/module/http/_meta/fields.yml @@ -13,7 +13,7 @@ HTTP request information fields: - name: header - type: nested + type: object description: > The HTTP headers sent - name: method @@ -30,7 +30,7 @@ HTTP response information fields: - name: header - type: nested + type: object description: > The HTTP headers received - name: status_code From d0f9e19d219a1984fabe0cd299f5a47e41aef8fa Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Fri, 29 Sep 2017 13:43:46 +0200 Subject: [PATCH 56/96] Move Field(s) to common. (#5271) Make type Field(s) reusable and seperate from transformation logic. --- libbeat/common/field.go | 116 ++++++++++ libbeat/common/field_test.go | 117 +++++++++++ libbeat/template/field.go | 238 --------------------- libbeat/template/field_test.go | 325 ----------------------------- libbeat/template/fields.go | 120 ----------- libbeat/template/fields_test.go | 110 ---------- libbeat/template/processor.go | 259 +++++++++++++++++++++++ libbeat/template/processor_test.go | 284 +++++++++++++++++++++++++ libbeat/template/template.go | 23 +- 9 files changed, 779 insertions(+), 813 deletions(-) create mode 100644 libbeat/common/field.go create mode 100644 libbeat/common/field_test.go delete mode 100644 libbeat/template/field.go delete mode 100644 libbeat/template/field_test.go delete mode 100644 libbeat/template/fields.go delete mode 100644 libbeat/template/fields_test.go create mode 100644 libbeat/template/processor.go create mode 100644 libbeat/template/processor_test.go diff --git a/libbeat/common/field.go b/libbeat/common/field.go new file mode 100644 index 00000000000..0cbafb37161 --- /dev/null +++ b/libbeat/common/field.go @@ -0,0 +1,116 @@ +package common + +import ( + "fmt" + "strings" + + "github.com/elastic/go-ucfg/yaml" +) + +//This reflects allowed attributes for field definitions in the fields.yml. +//No logic is put into this data structure. +//The purpose is to enable using different kinds of transformation, on top of the same data structure. +//Current transformation: +// -ElasticSearch Template +//Future: +// -Kibana Index Pattern + +type Field struct { + Name string `config:"name"` + Type string `config:"type"` + Description string `config:"description"` + Format string `config:"format"` + ScalingFactor int `config:"scaling_factor"` + Fields Fields `config:"fields"` + MultiFields Fields `config:"multi_fields"` + ObjectType string `config:"object_type"` + Enabled *bool `config:"enabled"` + Analyzer string `config:"analyzer"` + SearchAnalyzer string `config:"search_analyzer"` + Norms bool `config:"norms"` + Dynamic DynamicType `config:"dynamic"` + Index *bool `config:"index"` + DocValues *bool `config:"doc_values"` + + Path string +} + +type Fields []Field + +func LoadFieldsYaml(path string) (Fields, error) { + keys := []Field{} + + cfg, err := yaml.NewConfigWithFile(path) + if err != nil { + return nil, err + } + cfg.Unpack(&keys) + + fields := Fields{} + + for _, key := range keys { + fields = append(fields, key.Fields...) + } + return fields, nil +} + +// HasKey checks if inside fields the given key exists +// The key can be in the form of a.b.c and it will check if the nested field exist +// In case the key is `a` and there is a value `a.b` false is return as it only +// returns true if it's a leave node +func (f Fields) HasKey(key string) bool { + keys := strings.Split(key, ".") + return f.hasKey(keys) +} + +// Recursively generates the correct key based on the dots +// The mapping requires "properties" between each layer. This is added here. +func GenerateKey(key string) string { + if strings.Contains(key, ".") { + keys := strings.SplitN(key, ".", 2) + key = keys[0] + ".properties." + GenerateKey(keys[1]) + } + return key +} + +type DynamicType struct{ Value interface{} } + +func (d *DynamicType) Unpack(s string) error { + switch s { + case "true": + d.Value = true + case "false": + d.Value = false + case "strict": + d.Value = s + default: + return fmt.Errorf("'%v' is invalid dynamic setting", s) + } + return nil +} + +func (f Fields) hasKey(keys []string) bool { + // Nothing to compare anymore + if len(keys) == 0 { + return false + } + + key := keys[0] + keys = keys[1:] + + for _, field := range f { + if field.Name == key { + + if len(field.Fields) > 0 { + return field.Fields.hasKey(keys) + } + // Last entry in the tree but still more keys + if len(keys) > 0 { + return false + } + + return true + } + } + return false +} diff --git a/libbeat/common/field_test.go b/libbeat/common/field_test.go new file mode 100644 index 00000000000..2bab75a67d6 --- /dev/null +++ b/libbeat/common/field_test.go @@ -0,0 +1,117 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/go-ucfg/yaml" +) + +func TestFieldsHasKey(t *testing.T) { + tests := []struct { + key string + fields Fields + result bool + }{ + { + key: "test.find", + fields: Fields{}, + result: false, + }, + { + key: "test.find", + fields: Fields{ + Field{Name: "test"}, + Field{Name: "find"}, + }, + result: false, + }, + { + key: "test.find", + fields: Fields{ + Field{ + Name: "test", Fields: Fields{ + Field{ + Name: "find", + }, + }, + }, + }, + result: true, + }, + { + key: "test", + fields: Fields{ + Field{ + Name: "test", Fields: Fields{ + Field{ + Name: "find", + }, + }, + }, + }, + result: false, + }, + } + + for _, test := range tests { + assert.Equal(t, test.result, test.fields.HasKey(test.key)) + } +} + +func TestDynamicYaml(t *testing.T) { + tests := []struct { + input []byte + output Field + error bool + }{ + { + input: []byte(` +name: test +dynamic: true`), + output: Field{ + Name: "test", + Dynamic: DynamicType{true}, + }, + }, + { + input: []byte(` +name: test +dynamic: "true"`), + output: Field{ + Name: "test", + Dynamic: DynamicType{true}, + }, + }, + { + input: []byte(` +name: test +dynamic: "blue"`), + error: true, + }, + { + input: []byte(` +name: test +dynamic: "strict"`), + output: Field{ + Name: "test", + Dynamic: DynamicType{"strict"}, + }, + }, + } + + for _, test := range tests { + keys := Field{} + + cfg, err := yaml.NewConfig(test.input) + assert.NoError(t, err) + err = cfg.Unpack(&keys) + + if err != nil { + assert.True(t, test.error) + } else { + assert.Equal(t, test.output.Dynamic, keys.Dynamic) + } + } +} diff --git a/libbeat/template/field.go b/libbeat/template/field.go deleted file mode 100644 index a2a401ffe21..00000000000 --- a/libbeat/template/field.go +++ /dev/null @@ -1,238 +0,0 @@ -package template - -import ( - "fmt" - "strings" - - "github.com/elastic/beats/libbeat/common" -) - -var ( - defaultScalingFactor = 1000 -) - -type Field struct { - Name string `config:"name"` - Type string `config:"type"` - Description string `config:"description"` - Format string `config:"format"` - ScalingFactor int `config:"scaling_factor"` - Fields Fields `config:"fields"` - MultiFields Fields `config:"multi_fields"` - ObjectType string `config:"object_type"` - Enabled *bool `config:"enabled"` - Analyzer string `config:"analyzer"` - SearchAnalyzer string `config:"search_analyzer"` - Norms bool `config:"norms"` - Dynamic dynamicType `config:"dynamic"` - Index *bool `config:"index"` - DocValues *bool `config:"doc_values"` - - path string - esVersion common.Version -} - -// This includes all entries without special handling for different versions. -// Currently this is: -// long, geo_point, date, short, byte, float, double, boolean -func (f *Field) other() common.MapStr { - property := f.getDefaultProperties() - if f.Type != "" { - property["type"] = f.Type - } - - return property -} - -func (f *Field) integer() common.MapStr { - property := f.getDefaultProperties() - property["type"] = "long" - return property -} - -func (f *Field) scaledFloat() common.MapStr { - property := f.getDefaultProperties() - property["type"] = "scaled_float" - - if f.esVersion.IsMajor(2) { - property["type"] = "float" - } else { - // Set default scaling factor - if f.ScalingFactor == 0 { - f.ScalingFactor = defaultScalingFactor - } - property["scaling_factor"] = f.ScalingFactor - } - return property -} - -func (f *Field) halfFloat() common.MapStr { - property := f.getDefaultProperties() - property["type"] = "half_float" - - if f.esVersion.IsMajor(2) { - property["type"] = "float" - } - return property -} - -func (f *Field) ip() common.MapStr { - property := f.getDefaultProperties() - - property["type"] = "ip" - - if f.esVersion.IsMajor(2) { - property["type"] = "string" - property["ignore_above"] = 1024 - property["index"] = "not_analyzed" - } - return property -} - -func (f *Field) keyword() common.MapStr { - property := f.getDefaultProperties() - - property["type"] = "keyword" - property["ignore_above"] = 1024 - - if f.esVersion.IsMajor(2) { - property["type"] = "string" - property["ignore_above"] = 1024 - property["index"] = "not_analyzed" - } - return property -} - -func (f *Field) text() common.MapStr { - properties := f.getDefaultProperties() - - properties["type"] = "text" - - if f.esVersion.IsMajor(2) { - properties["type"] = "string" - properties["index"] = "analyzed" - if !f.Norms { - properties["norms"] = common.MapStr{ - "enabled": false, - } - } - } else { - if !f.Norms { - properties["norms"] = false - } - } - - if f.Analyzer != "" { - properties["analyzer"] = f.Analyzer - } - - if f.SearchAnalyzer != "" { - properties["search_analyzer"] = f.SearchAnalyzer - } - - if len(f.MultiFields) > 0 { - fields := common.MapStr{} - f.MultiFields.process("", f.esVersion, fields) - properties["fields"] = fields - } - - return properties -} - -func (f *Field) array() common.MapStr { - properties := f.getDefaultProperties() - if f.ObjectType != "" { - properties["type"] = f.ObjectType - } - return properties -} - -func (f *Field) object() common.MapStr { - dynProperties := f.getDefaultProperties() - - switch f.ObjectType { - case "text": - dynProperties["type"] = "text" - - if f.esVersion.IsMajor(2) { - dynProperties["type"] = "string" - dynProperties["index"] = "analyzed" - } - f.addDynamicTemplate(dynProperties, "string") - case "long": - dynProperties["type"] = f.ObjectType - f.addDynamicTemplate(dynProperties, "long") - case "keyword": - dynProperties["type"] = f.ObjectType - f.addDynamicTemplate(dynProperties, "string") - } - - properties := f.getDefaultProperties() - properties["type"] = "object" - if f.Enabled != nil { - properties["enabled"] = *f.Enabled - } - - if f.Dynamic.value != nil { - properties["dynamic"] = f.Dynamic.value - } - - return properties -} - -func (f *Field) addDynamicTemplate(properties common.MapStr, matchType string) { - path := "" - if len(f.path) > 0 { - path = f.path + "." - } - template := common.MapStr{ - // Set the path of the field as name - path + f.Name: common.MapStr{ - "mapping": properties, - "match_mapping_type": matchType, - "path_match": path + f.Name + ".*", - }, - } - - dynamicTemplates = append(dynamicTemplates, template) -} - -func (f *Field) getDefaultProperties() common.MapStr { - // Currently no defaults exist - properties := common.MapStr{} - - if f.Index != nil { - properties["index"] = *f.Index - } - - if f.DocValues != nil { - properties["doc_values"] = *f.DocValues - } - return properties -} - -// Recursively generates the correct key based on the dots -// The mapping requires "properties" between each layer. This is added here. -func generateKey(key string) string { - if strings.Contains(key, ".") { - keys := strings.SplitN(key, ".", 2) - key = keys[0] + ".properties." + generateKey(keys[1]) - } - return key -} - -type dynamicType struct{ value interface{} } - -func (d *dynamicType) Unpack(s string) error { - switch s { - case "true": - d.value = true - case "false": - d.value = false - case "strict": - d.value = s - default: - return fmt.Errorf("'%v' is invalid dynamic setting", s) - } - return nil -} diff --git a/libbeat/template/field_test.go b/libbeat/template/field_test.go deleted file mode 100644 index 6a5e56e6fd8..00000000000 --- a/libbeat/template/field_test.go +++ /dev/null @@ -1,325 +0,0 @@ -package template - -import ( - "testing" - - "github.com/elastic/beats/libbeat/common" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/go-ucfg/yaml" -) - -func TestField(t *testing.T) { - esVersion2, err := common.NewVersion("2.0.0") - assert.NoError(t, err) - - falseVar := false - trueVar := true - - tests := []struct { - field Field - method func(f Field) common.MapStr - output common.MapStr - }{ - { - field: Field{Type: "long"}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "long", - }, - }, - { - field: Field{Type: "scaled_float"}, - method: func(f Field) common.MapStr { return f.scaledFloat() }, - output: common.MapStr{ - "type": "scaled_float", - "scaling_factor": 1000, - }, - }, - { - field: Field{Type: "scaled_float", ScalingFactor: 100}, - method: func(f Field) common.MapStr { return f.scaledFloat() }, - output: common.MapStr{ - "type": "scaled_float", - "scaling_factor": 100, - }, - }, - { - field: Field{Type: "scaled_float", esVersion: *esVersion2}, - method: func(f Field) common.MapStr { return f.scaledFloat() }, - output: common.MapStr{ - "type": "float", - }, - }, - { - field: Field{Type: "object", Enabled: &falseVar}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "type": "object", - "enabled": false, - }, - }, - { - field: Field{Type: "array"}, - method: func(f Field) common.MapStr { return f.array() }, - output: common.MapStr{}, - }, - { - field: Field{Type: "array", ObjectType: "text"}, - method: func(f Field) common.MapStr { return f.array() }, - output: common.MapStr{"type": "text"}, - }, - { - field: Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}, - method: func(f Field) common.MapStr { return f.array() }, - output: common.MapStr{"index": false, "type": "keyword"}, - }, - { - field: Field{Type: "object", Enabled: &falseVar}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "type": "object", - "enabled": false, - }, - }, - { - field: Field{Type: "text", Analyzer: "autocomplete"}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "analyzer": "autocomplete", - "norms": false, - }, - }, - { - field: Field{Type: "text", Analyzer: "autocomplete", Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "analyzer": "autocomplete", - }, - }, - { - field: Field{Type: "text", SearchAnalyzer: "standard", Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "search_analyzer": "standard", - }, - }, - { - field: Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "analyzer": "autocomplete", - "search_analyzer": "standard", - }, - }, - { - field: Field{Type: "text", MultiFields: Fields{Field{Name: "raw", Type: "keyword"}}, Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "fields": common.MapStr{ - "raw": common.MapStr{ - "type": "keyword", - "ignore_above": 1024, - }, - }, - }, - }, - { - field: Field{Type: "text", MultiFields: Fields{ - Field{Name: "raw", Type: "keyword"}, - Field{Name: "indexed", Type: "text"}, - }, Norms: true}, - method: func(f Field) common.MapStr { return f.text() }, - output: common.MapStr{ - "type": "text", - "fields": common.MapStr{ - "raw": common.MapStr{ - "type": "keyword", - "ignore_above": 1024, - }, - "indexed": common.MapStr{ - "type": "text", - "norms": false, - }, - }, - }, - }, - { - field: Field{Dynamic: dynamicType{false}}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "dynamic": false, "type": "object", - }, - }, - { - field: Field{Dynamic: dynamicType{true}}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "dynamic": true, "type": "object", - }, - }, - { - field: Field{Dynamic: dynamicType{"strict"}}, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "dynamic": "strict", "type": "object", - }, - }, - { - field: Field{Type: "long", Index: &falseVar}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "long", "index": false, - }, - }, - { - field: Field{Type: "text", Index: &trueVar}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "text", "index": true, - }, - }, - { - field: Field{Type: "long", DocValues: &falseVar}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "long", "doc_values": false, - }, - }, - { - field: Field{Type: "text", DocValues: &trueVar}, - method: func(f Field) common.MapStr { return f.other() }, - output: common.MapStr{ - "type": "text", "doc_values": true, - }, - }, - } - - for _, test := range tests { - output := test.method(test.field) - assert.Equal(t, test.output, output) - } -} - -func TestDynamicTemplate(t *testing.T) { - tests := []struct { - field Field - method func(f Field) common.MapStr - output common.MapStr - }{ - { - field: Field{ - Type: "object", ObjectType: "keyword", - Name: "context", - }, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "context": common.MapStr{ - "mapping": common.MapStr{"type": "keyword"}, - "match_mapping_type": "string", - "path_match": "context.*", - }, - }, - }, - { - field: Field{ - Type: "object", ObjectType: "long", - path: "language", Name: "english", - }, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "long"}, - "match_mapping_type": "long", - "path_match": "language.english.*", - }, - }, - }, - { - field: Field{ - Type: "object", ObjectType: "text", - path: "language", Name: "english", - }, - method: func(f Field) common.MapStr { return f.object() }, - output: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "text"}, - "match_mapping_type": "string", - "path_match": "language.english.*", - }, - }, - }, - } - - for _, test := range tests { - dynamicTemplates = nil - test.method(test.field) - assert.Equal(t, test.output, dynamicTemplates[0]) - } -} - -func TestDynamicYaml(t *testing.T) { - tests := []struct { - input []byte - output Field - error bool - }{ - { - input: []byte(` -name: test -dynamic: true -`), - output: Field{ - Name: "test", - Dynamic: dynamicType{true}, - }, - }, - { - input: []byte(` -name: test -dynamic: "true" -`), - output: Field{ - Name: "test", - Dynamic: dynamicType{true}, - }, - }, - { - input: []byte(` -name: test -dynamic: "blue" -`), - error: true, - }, - { - input: []byte(` -name: test -dynamic: "strict" -`), - output: Field{ - Name: "test", - Dynamic: dynamicType{"strict"}, - }, - }, - } - - for _, test := range tests { - keys := Field{} - - cfg, err := yaml.NewConfig(test.input) - assert.NoError(t, err) - err = cfg.Unpack(&keys) - - if err != nil { - assert.True(t, test.error) - } else { - assert.Equal(t, test.output.Dynamic, keys.Dynamic) - } - } -} diff --git a/libbeat/template/fields.go b/libbeat/template/fields.go deleted file mode 100644 index 937c0f6a398..00000000000 --- a/libbeat/template/fields.go +++ /dev/null @@ -1,120 +0,0 @@ -package template - -import ( - "errors" - "strings" - - "github.com/elastic/beats/libbeat/common" -) - -var ( - defaultType = "keyword" -) - -type Fields []Field - -func (f Fields) process(path string, esVersion common.Version, output common.MapStr) error { - for _, field := range f { - - var mapping common.MapStr - field.path = path - field.esVersion = esVersion - - // If not type is defined, it assumes keyword - if field.Type == "" { - field.Type = defaultType - } - - switch field.Type { - case "ip": - mapping = field.ip() - case "scaled_float": - mapping = field.scaledFloat() - case "half_float": - mapping = field.halfFloat() - case "integer": - mapping = field.integer() - case "text": - mapping = field.text() - case "keyword": - mapping = field.keyword() - case "object": - mapping = field.object() - case "array": - mapping = field.array() - case "group": - var newPath string - if path == "" { - newPath = field.Name - } else { - newPath = path + "." + field.Name - } - mapping = common.MapStr{} - if field.Dynamic.value != nil { - mapping["dynamic"] = field.Dynamic.value - } - - // Combine properties with previous field definitions (if any) - properties := common.MapStr{} - key := generateKey(field.Name) + ".properties" - currentProperties, err := output.GetValue(key) - if err == nil { - var ok bool - properties, ok = currentProperties.(common.MapStr) - if !ok { - // This should never happen - return errors.New(key + " is expected to be a MapStr") - } - } - - if err := field.Fields.process(newPath, esVersion, properties); err != nil { - return err - } - mapping["properties"] = properties - - default: - mapping = field.other() - } - - if len(mapping) > 0 { - output.Put(generateKey(field.Name), mapping) - } - } - - return nil -} - -// HasKey checks if inside fields the given key exists -// The key can be in the form of a.b.c and it will check if the nested field exist -// In case the key is `a` and there is a value `a.b` false is return as it only -// returns true if it's a leave node -func (f Fields) HasKey(key string) bool { - keys := strings.Split(key, ".") - return f.hasKey(keys) -} - -func (f Fields) hasKey(keys []string) bool { - // Nothing to compare anymore - if len(keys) == 0 { - return false - } - - key := keys[0] - keys = keys[1:] - - for _, field := range f { - if field.Name == key { - - if len(field.Fields) > 0 { - return field.Fields.hasKey(keys) - } - // Last entry in the tree but still more keys - if len(keys) > 0 { - return false - } - - return true - } - } - return false -} diff --git a/libbeat/template/fields_test.go b/libbeat/template/fields_test.go deleted file mode 100644 index d39fa131afc..00000000000 --- a/libbeat/template/fields_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package template - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/common" -) - -func TestHasKey(t *testing.T) { - tests := []struct { - key string - fields Fields - result bool - }{ - { - key: "test.find", - fields: Fields{}, - result: false, - }, - { - key: "test.find", - fields: Fields{ - Field{Name: "test"}, - Field{Name: "find"}, - }, - result: false, - }, - { - key: "test.find", - fields: Fields{ - Field{ - Name: "test", Fields: Fields{ - Field{ - Name: "find", - }, - }, - }, - }, - result: true, - }, - { - key: "test", - fields: Fields{ - Field{ - Name: "test", Fields: Fields{ - Field{ - Name: "find", - }, - }, - }, - }, - result: false, - }, - } - - for _, test := range tests { - assert.Equal(t, test.result, test.fields.HasKey(test.key)) - } -} - -func TestPropertiesCombine(t *testing.T) { - // Test common fields are combined even if they come from different objects - fields := Fields{ - Field{ - Name: "test", - Type: "group", - Fields: Fields{ - Field{ - Name: "one", - Type: "text", - }, - }, - }, - Field{ - Name: "test", - Type: "group", - Fields: Fields{ - Field{ - Name: "two", - Type: "text", - }, - }, - }, - } - - output := common.MapStr{} - version, err := common.NewVersion("6.0.0") - if err != nil { - t.Fatal(err) - } - - err = fields.process("", *version, output) - if err != nil { - t.Fatal(err) - } - - v1, err := output.GetValue("test.properties.one") - if err != nil { - t.Fatal(err) - } - v2, err := output.GetValue("test.properties.two") - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, v1, common.MapStr{"type": "text", "norms": false}) - assert.Equal(t, v2, common.MapStr{"type": "text", "norms": false}) -} diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go new file mode 100644 index 00000000000..af9a6f73d5c --- /dev/null +++ b/libbeat/template/processor.go @@ -0,0 +1,259 @@ +package template + +import ( + "errors" + + "github.com/elastic/beats/libbeat/common" +) + +type Processor struct { + EsVersion common.Version +} + +var ( + defaultScalingFactor = 1000 +) + +// This includes all entries without special handling for different versions. +// Currently this is: +// long, geo_point, date, short, byte, float, double, boolean +func (p *Processor) process(fields common.Fields, path string, output common.MapStr) error { + for _, field := range fields { + + field.Path = path + var mapping common.MapStr + + switch field.Type { + case "ip": + mapping = p.ip(&field) + case "scaled_float": + mapping = p.scaledFloat(&field) + case "half_float": + mapping = p.halfFloat(&field) + case "integer": + mapping = p.integer(&field) + case "text": + mapping = p.text(&field) + case "", "keyword": + mapping = p.keyword(&field) + case "object": + mapping = p.object(&field) + case "array": + mapping = p.array(&field) + case "group": + var newPath string + if path == "" { + newPath = field.Name + } else { + newPath = path + "." + field.Name + } + mapping = common.MapStr{} + if field.Dynamic.Value != nil { + mapping["dynamic"] = field.Dynamic.Value + } + + // Combine properties with previous field definitions (if any) + properties := common.MapStr{} + key := common.GenerateKey(field.Name) + ".properties" + currentProperties, err := output.GetValue(key) + if err == nil { + var ok bool + properties, ok = currentProperties.(common.MapStr) + if !ok { + // This should never happen + return errors.New(key + " is expected to be a MapStr") + } + } + + if err := p.process(field.Fields, newPath, properties); err != nil { + return err + } + mapping["properties"] = properties + + default: + mapping = p.other(&field) + } + + if len(mapping) > 0 { + output.Put(common.GenerateKey(field.Name), mapping) + } + } + return nil +} + +func (p *Processor) other(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + if f.Type != "" { + property["type"] = f.Type + } + + return property +} + +func (p *Processor) integer(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + property["type"] = "long" + return property +} + +func (p *Processor) scaledFloat(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + property["type"] = "scaled_float" + + if p.EsVersion.IsMajor(2) { + property["type"] = "float" + } else { + scalingFactor := f.ScalingFactor + // Set default scaling factor + if scalingFactor == 0 { + scalingFactor = defaultScalingFactor + } + property["scaling_factor"] = scalingFactor + } + return property +} + +func (p *Processor) halfFloat(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + property["type"] = "half_float" + + if p.EsVersion.IsMajor(2) { + property["type"] = "float" + } + return property +} + +func (p *Processor) ip(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + + property["type"] = "ip" + + if p.EsVersion.IsMajor(2) { + property["type"] = "string" + property["ignore_above"] = 1024 + property["index"] = "not_analyzed" + } + return property +} + +func (p *Processor) keyword(f *common.Field) common.MapStr { + property := getDefaultProperties(f) + + property["type"] = "keyword" + property["ignore_above"] = 1024 + + if p.EsVersion.IsMajor(2) { + property["type"] = "string" + property["ignore_above"] = 1024 + property["index"] = "not_analyzed" + } + return property +} + +func (p *Processor) text(f *common.Field) common.MapStr { + properties := getDefaultProperties(f) + + properties["type"] = "text" + + if p.EsVersion.IsMajor(2) { + properties["type"] = "string" + properties["index"] = "analyzed" + if !f.Norms { + properties["norms"] = common.MapStr{ + "enabled": false, + } + } + } else { + if !f.Norms { + properties["norms"] = false + } + } + + if f.Analyzer != "" { + properties["analyzer"] = f.Analyzer + } + + if f.SearchAnalyzer != "" { + properties["search_analyzer"] = f.SearchAnalyzer + } + + if len(f.MultiFields) > 0 { + fields := common.MapStr{} + p.process(f.MultiFields, "", fields) + properties["fields"] = fields + } + + return properties +} + +func (p *Processor) array(f *common.Field) common.MapStr { + properties := getDefaultProperties(f) + if f.ObjectType != "" { + properties["type"] = f.ObjectType + } + return properties +} + +func (p *Processor) object(f *common.Field) common.MapStr { + dynProperties := getDefaultProperties(f) + + switch f.ObjectType { + case "text": + dynProperties["type"] = "text" + + if p.EsVersion.IsMajor(2) { + dynProperties["type"] = "string" + dynProperties["index"] = "analyzed" + } + addDynamicTemplate(f, dynProperties, "string") + case "long": + dynProperties["type"] = f.ObjectType + addDynamicTemplate(f, dynProperties, "long") + case "keyword": + dynProperties["type"] = f.ObjectType + addDynamicTemplate(f, dynProperties, "string") + } + + properties := getDefaultProperties(f) + properties["type"] = "object" + if f.Enabled != nil { + properties["enabled"] = *f.Enabled + } + + if f.Dynamic.Value != nil { + properties["dynamic"] = f.Dynamic.Value + } + + return properties +} + +func addDynamicTemplate(f *common.Field, properties common.MapStr, matchType string) { + path := "" + if len(f.Path) > 0 { + path = f.Path + "." + } + template := common.MapStr{ + // Set the path of the field as name + path + f.Name: common.MapStr{ + "mapping": properties, + "match_mapping_type": matchType, + "path_match": path + f.Name + ".*", + }, + } + + dynamicTemplates = append(dynamicTemplates, template) +} + +func getDefaultProperties(f *common.Field) common.MapStr { + // Currently no defaults exist + properties := common.MapStr{} + + if f.Index != nil { + properties["index"] = *f.Index + } + + if f.DocValues != nil { + properties["doc_values"] = *f.DocValues + } + return properties +} diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go new file mode 100644 index 00000000000..45133e3dce5 --- /dev/null +++ b/libbeat/template/processor_test.go @@ -0,0 +1,284 @@ +package template + +import ( + "testing" + + "github.com/elastic/beats/libbeat/common" + + "github.com/stretchr/testify/assert" +) + +func TestProcessor(t *testing.T) { + esVersion2, err := common.NewVersion("2.0.0") + assert.NoError(t, err) + + falseVar := false + trueVar := true + p := &Processor{} + pEsVersion2 := &Processor{EsVersion: *esVersion2} + + tests := []struct { + output common.MapStr + expected common.MapStr + }{ + { + output: p.other(&common.Field{Type: "long"}), + expected: common.MapStr{"type": "long"}, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float"}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 1000, + }, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float", ScalingFactor: 100}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 100, + }, + }, + { + output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}), + expected: common.MapStr{"type": "float"}, + }, + { + output: p.object(&common.Field{Type: "object", Enabled: &falseVar}), + expected: common.MapStr{ + "type": "object", + "enabled": false, + }, + }, + { + output: p.array(&common.Field{Type: "array"}), + expected: common.MapStr{}, + }, + { + output: p.array(&common.Field{Type: "array", ObjectType: "text"}), + expected: common.MapStr{"type": "text"}, + }, + { + output: p.array(&common.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), + expected: common.MapStr{"index": false, "type": "keyword"}, + }, + { + output: p.object(&common.Field{Type: "object", Enabled: &falseVar}), + expected: common.MapStr{ + "type": "object", + "enabled": false, + }, + }, + { + output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete"}), + expected: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + "norms": false, + }, + }, + { + output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete", Norms: true}), + expected: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + }, + }, + { + output: p.text(&common.Field{Type: "text", SearchAnalyzer: "standard", Norms: true}), + expected: common.MapStr{ + "type": "text", + "search_analyzer": "standard", + }, + }, + { + output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}), + expected: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + "search_analyzer": "standard", + }, + }, + { + output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{common.Field{Name: "raw", Type: "keyword"}}, Norms: true}), + expected: common.MapStr{ + "type": "text", + "fields": common.MapStr{ + "raw": common.MapStr{ + "type": "keyword", + "ignore_above": 1024, + }, + }, + }, + }, + { + output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{ + common.Field{Name: "raw", Type: "keyword"}, + common.Field{Name: "indexed", Type: "text"}, + }, Norms: true}), + expected: common.MapStr{ + "type": "text", + "fields": common.MapStr{ + "raw": common.MapStr{ + "type": "keyword", + "ignore_above": 1024, + }, + "indexed": common.MapStr{ + "type": "text", + "norms": false, + }, + }, + }, + }, + { + output: p.object(&common.Field{Dynamic: common.DynamicType{false}}), + expected: common.MapStr{ + "dynamic": false, "type": "object", + }, + }, + { + output: p.object(&common.Field{Dynamic: common.DynamicType{true}}), + expected: common.MapStr{ + "dynamic": true, "type": "object", + }, + }, + { + output: p.object(&common.Field{Dynamic: common.DynamicType{"strict"}}), + expected: common.MapStr{ + "dynamic": "strict", "type": "object", + }, + }, + { + output: p.other(&common.Field{Type: "long", Index: &falseVar}), + expected: common.MapStr{ + "type": "long", "index": false, + }, + }, + { + output: p.other(&common.Field{Type: "text", Index: &trueVar}), + expected: common.MapStr{ + "type": "text", "index": true, + }, + }, + { + output: p.other(&common.Field{Type: "long", DocValues: &falseVar}), + expected: common.MapStr{ + "type": "long", "doc_values": false, + }, + }, + { + output: p.other(&common.Field{Type: "text", DocValues: &trueVar}), + expected: common.MapStr{ + "type": "text", "doc_values": true, + }, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expected, test.output) + } +} + +func TestDynamicTemplate(t *testing.T) { + p := &Processor{} + tests := []struct { + field common.Field + expected common.MapStr + }{ + { + field: common.Field{ + Type: "object", ObjectType: "keyword", + Name: "context", + }, + expected: common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "keyword"}, + "match_mapping_type": "string", + "path_match": "context.*", + }, + }, + }, + { + field: common.Field{ + Type: "object", ObjectType: "long", + Path: "language", Name: "english", + }, + expected: common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "long"}, + "match_mapping_type": "long", + "path_match": "language.english.*", + }, + }, + }, + { + field: common.Field{ + Type: "object", ObjectType: "text", + Path: "language", Name: "english", + }, + expected: common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "text"}, + "match_mapping_type": "string", + "path_match": "language.english.*", + }, + }, + }, + } + + for _, test := range tests { + dynamicTemplates = nil + p.object(&test.field) + assert.Equal(t, test.expected, dynamicTemplates[0]) + } +} + +func TestPropertiesCombine(t *testing.T) { + // Test common fields are combined even if they come from different objects + fields := common.Fields{ + common.Field{ + Name: "test", + Type: "group", + Fields: common.Fields{ + common.Field{ + Name: "one", + Type: "text", + }, + }, + }, + common.Field{ + Name: "test", + Type: "group", + Fields: common.Fields{ + common.Field{ + Name: "two", + Type: "text", + }, + }, + }, + } + + output := common.MapStr{} + version, err := common.NewVersion("6.0.0") + if err != nil { + t.Fatal(err) + } + + p := Processor{EsVersion: *version} + err = p.process(fields, "", output) + if err != nil { + t.Fatal(err) + } + + v1, err := output.GetValue("test.properties.one") + if err != nil { + t.Fatal(err) + } + v2, err := output.GetValue("test.properties.two") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, v1, common.MapStr{"type": "text", "norms": false}) + assert.Equal(t, v2, common.MapStr{"type": "text", "norms": false}) +} diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 646614f6fc3..81b4148ef4d 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -7,7 +7,6 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/fmtstr" - "github.com/elastic/go-ucfg/yaml" ) var ( @@ -93,14 +92,15 @@ func New(beatVersion string, beatName string, esVersion string, config TemplateC // Load the given input and generates the input based on it func (t *Template) Load(file string) (common.MapStr, error) { - fields, err := loadYaml(file) + fields, err := common.LoadFieldsYaml(file) if err != nil { return nil, err } // Start processing at the root properties := common.MapStr{} - if err := fields.process("", t.esVersion, properties); err != nil { + processor := Processor{EsVersion: t.esVersion} + if err := processor.process(fields, "", properties); err != nil { return nil, err } output := t.generate(properties, dynamicTemplates) @@ -192,20 +192,3 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. return basicStructure } - -func loadYaml(path string) (Fields, error) { - keys := []Field{} - - cfg, err := yaml.NewConfigWithFile(path) - if err != nil { - return nil, err - } - cfg.Unpack(&keys) - - fields := Fields{} - - for _, key := range keys { - fields = append(fields, key.Fields...) - } - return fields, nil -} From e912823418affed6c93cbdd2c72fea1d3575ce71 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 2 Oct 2017 09:25:34 +0200 Subject: [PATCH 57/96] Fix mapping for ceph num_remapped_pgs field (#5273) The field num_remapped_pgs was mapped as num_in_osds which lead to a field defined twice and the mapping of the field did not exist. --- metricbeat/docs/fields.asciidoc | 2 +- metricbeat/module/ceph/cluster_status/_meta/fields.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index a9138878fb1..5088d17e4f5 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1065,7 +1065,7 @@ Shows how many osds are on the state of IN [float] -=== `ceph.cluster_status.osd.num_in_osds` +=== `ceph.cluster_status.osd.num_remapped_pgs` type: long diff --git a/metricbeat/module/ceph/cluster_status/_meta/fields.yml b/metricbeat/module/ceph/cluster_status/_meta/fields.yml index 5453ac7beee..59cb57dcedf 100644 --- a/metricbeat/module/ceph/cluster_status/_meta/fields.yml +++ b/metricbeat/module/ceph/cluster_status/_meta/fields.yml @@ -103,11 +103,11 @@ type: long description: > Shows how many osds are on the state of IN - - name: osd.num_in_osds + - name: osd.num_remapped_pgs type: long description: > Shows how many osds are on the state of REMAPPED - name: osd.epoch type: long description: > - epoch number \ No newline at end of file + epoch number From 247ee165206410dd2cf0f919f208b1a0d0523d88 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 2 Oct 2017 09:35:27 +0200 Subject: [PATCH 58/96] Update latest environment to 6.0.0-rc1 (#5274) Fix sha1 sum to sha512 --- .../environments/docker/elasticsearch/Dockerfile-snapshot | 4 ++-- testing/environments/latest.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/testing/environments/docker/elasticsearch/Dockerfile-snapshot b/testing/environments/docker/elasticsearch/Dockerfile-snapshot index 2491e558254..afcaa49be30 100644 --- a/testing/environments/docker/elasticsearch/Dockerfile-snapshot +++ b/testing/environments/docker/elasticsearch/Dockerfile-snapshot @@ -21,8 +21,8 @@ WORKDIR /usr/share/elasticsearch # Download/extract defined ES version. busybox tar can't strip leading dir. RUN curl -L -o elasticsearch-${ELASTIC_VERSION}.tar.gz ${DOWNLOAD_URL}/elasticsearch/elasticsearch-${ELASTIC_VERSION}.tar.gz?c=${CACHE_BUST} && \ - EXPECTED_SHA=$(wget -O - ${DOWNLOAD_URL}/elasticsearch/elasticsearch-${ELASTIC_VERSION}.tar.gz.sha1) && \ - test $EXPECTED_SHA == $(sha1sum elasticsearch-${ELASTIC_VERSION}.tar.gz | awk '{print $1}') && \ + EXPECTED_SHA=$(wget -O - ${DOWNLOAD_URL}/elasticsearch/elasticsearch-${ELASTIC_VERSION}.tar.gz.sha512) && \ + test $EXPECTED_SHA == $(sha512sum elasticsearch-${ELASTIC_VERSION}.tar.gz | awk '{print $1}') && \ tar zxf elasticsearch-${ELASTIC_VERSION}.tar.gz && \ chown -R elasticsearch:elasticsearch elasticsearch-${ELASTIC_VERSION} && \ mv elasticsearch-${ELASTIC_VERSION}/* . && \ diff --git a/testing/environments/latest.yml b/testing/environments/latest.yml index 854ba41654d..9edd224102b 100644 --- a/testing/environments/latest.yml +++ b/testing/environments/latest.yml @@ -3,7 +3,7 @@ version: '2.1' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.0.0-beta1 + image: docker.elastic.co/elasticsearch/elasticsearch:6.0.0-rc1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] environment: @@ -18,13 +18,13 @@ services: context: docker/logstash dockerfile: Dockerfile args: - ELASTIC_VERSION: 6.0.0-beta1 + ELASTIC_VERSION: 6.0.0-rc1 DOWNLOAD_URL: https://artifacts.elastic.co/downloads environment: - ES_HOST=elasticsearch kibana: - image: docker.elastic.co/kibana/kibana:6.0.0-beta1 + image: docker.elastic.co/kibana/kibana:6.0.0-rc1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5601"] retries: 6 From 1751c269c0625b54d0adfdadd746a8e14aeb4f55 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Mon, 2 Oct 2017 10:33:07 +0200 Subject: [PATCH 59/96] Rewrite Generate-Index-Pattern in Go. (#5272) * Rewrite logic in go (instead of python) * Add tests and integration tests * Add existing metricbeat index pattern to ensure no logic is changed * Add unique key check. This is the only intended logic change. --- .../kibana/5.x/index-pattern/libbeat.json | 6 +- .../kibana/default/index-pattern/libbeat.json | 18 +- libbeat/common/field.go | 42 +- libbeat/kibana/index.go | 121 + libbeat/kibana/index_test.go | 174 + libbeat/kibana/testdata/fields.yml | 6858 +++++++++++++++++ .../kibana/testdata/metricbeat-5x-old.json | 6 + .../testdata/metricbeat-default-old.json | 16 + libbeat/kibana/transform.go | 132 + libbeat/kibana/transform_test.go | 279 + libbeat/scripts/Makefile | 2 +- libbeat/scripts/generate_index_pattern.go | 24 + 12 files changed, 7647 insertions(+), 31 deletions(-) create mode 100644 libbeat/kibana/index.go create mode 100644 libbeat/kibana/index_test.go create mode 100644 libbeat/kibana/testdata/fields.yml create mode 100644 libbeat/kibana/testdata/metricbeat-5x-old.json create mode 100644 libbeat/kibana/testdata/metricbeat-default-old.json create mode 100644 libbeat/kibana/transform.go create mode 100644 libbeat/kibana/transform_test.go create mode 100644 libbeat/scripts/generate_index_pattern.go diff --git a/libbeat/_meta/kibana/5.x/index-pattern/libbeat.json b/libbeat/_meta/kibana/5.x/index-pattern/libbeat.json index 2e32925e83a..dd8832c928e 100644 --- a/libbeat/_meta/kibana/5.x/index-pattern/libbeat.json +++ b/libbeat/_meta/kibana/5.x/index-pattern/libbeat.json @@ -1,6 +1,6 @@ { - "fields": "[{\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.timezone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"@timestamp\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"fields\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"error.message\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"error.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.provider\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.machine_type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.availability_zone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.project_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.region\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.image\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.namespace\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.annotations\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.image\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_id\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"_type\", \"searchable\": true, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_index\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_score\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"number\", \"scripted\": false}]", - "fieldFormatMap": "{\"@timestamp\": {\"id\": \"date\"}}", - "timeFieldName": "@timestamp", + "fieldFormatMap": "{\"@timestamp\":{\"id\":\"date\"}}", + "fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"beat.name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"beat.hostname\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"beat.timezone\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"beat.version\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"@timestamp\",\"scripted\":false,\"searchable\":true,\"type\":\"date\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"tags\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"fields\",\"scripted\":false,\"searchable\":true},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"error.message\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"error.code\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"error.type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.provider\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.instance_id\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.instance_name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.machine_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.availability_zone\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.project_id\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.region\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"docker.container.id\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"docker.container.image\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"docker.container.name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"docker.container.labels\",\"scripted\":false,\"searchable\":true},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.pod.name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.namespace\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.labels\",\"scripted\":false,\"searchable\":true},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.annotations\",\"scripted\":false,\"searchable\":true},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.container.name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.container.image\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"}]", + "timeFieldName": "@timestamp", "title": "libbeat-*" } \ No newline at end of file diff --git a/libbeat/_meta/kibana/default/index-pattern/libbeat.json b/libbeat/_meta/kibana/default/index-pattern/libbeat.json index e74e6a98b15..4a1f3e3a13f 100644 --- a/libbeat/_meta/kibana/default/index-pattern/libbeat.json +++ b/libbeat/_meta/kibana/default/index-pattern/libbeat.json @@ -1,16 +1,16 @@ { - "version": "7.0.0-alpha1", "objects": [ { "attributes": { - "fields": "[{\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.timezone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"@timestamp\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"fields\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"error.message\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"error.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.provider\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.machine_type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.availability_zone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.project_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.region\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.image\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.namespace\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.annotations\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.image\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_id\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"_type\", \"searchable\": true, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_index\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_score\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"number\", \"scripted\": false}]", - "fieldFormatMap": "{\"@timestamp\": {\"id\": \"date\"}}", - "timeFieldName": "@timestamp", + "fieldFormatMap": "{\"@timestamp\":{\"id\":\"date\"}}", + "fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"beat.name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"beat.hostname\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"beat.timezone\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"beat.version\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"@timestamp\",\"scripted\":false,\"searchable\":true,\"type\":\"date\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"tags\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"fields\",\"scripted\":false,\"searchable\":true},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"error.message\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"error.code\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"error.type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.provider\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.instance_id\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.instance_name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.machine_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.availability_zone\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.project_id\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"meta.cloud.region\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"docker.container.id\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"docker.container.image\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"docker.container.name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"docker.container.labels\",\"scripted\":false,\"searchable\":true},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.pod.name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.namespace\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.labels\",\"scripted\":false,\"searchable\":true},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.annotations\",\"scripted\":false,\"searchable\":true},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.container.name\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"kubernetes.container.image\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"}]", + "timeFieldName": "@timestamp", "title": "libbeat-*" - }, - "version": 1, - "type": "index-pattern", - "id": "libbeat-*" + }, + "id": "libbeat-*", + "type": "index-pattern", + "version": 1 } - ] + ], + "version": "7.0.0-alpha1" } \ No newline at end of file diff --git a/libbeat/common/field.go b/libbeat/common/field.go index 0cbafb37161..a29f668b344 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -12,9 +12,10 @@ import ( //The purpose is to enable using different kinds of transformation, on top of the same data structure. //Current transformation: // -ElasticSearch Template -//Future: // -Kibana Index Pattern +type Fields []Field + type Field struct { Name string `config:"name"` Type string `config:"type"` @@ -32,11 +33,32 @@ type Field struct { Index *bool `config:"index"` DocValues *bool `config:"doc_values"` + // Kibana specific + Pattern string `config:"pattern"` + Analyzed *bool `config:"analyzed"` + Count int `config:"count"` + Searchable *bool `config:"searchable"` + Aggregatable *bool `config:"aggregatable"` + InputFormat string `config:"input_format"` + Path string } -type Fields []Field +type DynamicType struct{ Value interface{} } +func (d *DynamicType) Unpack(s string) error { + switch s { + case "true": + d.Value = true + case "false": + d.Value = false + case "strict": + d.Value = s + default: + return fmt.Errorf("'%v' is invalid dynamic setting", s) + } + return nil +} func LoadFieldsYaml(path string) (Fields, error) { keys := []Field{} @@ -73,22 +95,6 @@ func GenerateKey(key string) string { return key } -type DynamicType struct{ Value interface{} } - -func (d *DynamicType) Unpack(s string) error { - switch s { - case "true": - d.Value = true - case "false": - d.Value = false - case "strict": - d.Value = s - default: - return fmt.Errorf("'%v' is invalid dynamic setting", s) - } - return nil -} - func (f Fields) hasKey(keys []string) bool { // Nothing to compare anymore if len(keys) == 0 { diff --git a/libbeat/kibana/index.go b/libbeat/kibana/index.go new file mode 100644 index 00000000000..68d033e2ef5 --- /dev/null +++ b/libbeat/kibana/index.go @@ -0,0 +1,121 @@ +package kibana + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "path/filepath" + "regexp" + + "github.com/elastic/beats/libbeat/common" +) + +type Index struct { + Version string + IndexName string + BeatDir string + BeatName string + + fieldsYaml string + targetDirDefault string + targetDir5x string + targetFilename string +} + +// Create the Index-Pattern for Kibana for 5.x and default. +func (i *Index) Create() ([]string, error) { + indices := []string{} + + err := i.init() + if err != nil { + return nil, err + } + + commonFields, err := common.LoadFieldsYaml(i.fieldsYaml) + if err != nil { + return nil, err + } + transformed := TransformFields("@timestamp", i.IndexName, commonFields) + + if fieldsBytes, err := json.Marshal(transformed["fields"]); err != nil { + return nil, err + } else { + transformed["fields"] = string(fieldsBytes) + } + if fieldFormatBytes, err := json.Marshal(transformed["fieldFormatMap"]); err != nil { + return nil, err + } else { + transformed["fieldFormatMap"] = string(fieldFormatBytes) + } + + file5x := filepath.Join(i.targetDir5x, i.targetFilename) + err = dumpToFile(file5x, transformed) + if err != nil { + return nil, err + } + indices = append(indices, file5x) + + out := common.MapStr{ + "version": i.Version, + "objects": []common.MapStr{ + common.MapStr{ + "type": "index-pattern", + "id": i.IndexName, + "version": 1, + "attributes": transformed, + }, + }, + } + + fileDefault := filepath.Join(i.targetDirDefault, i.targetFilename) + err = dumpToFile(fileDefault, out) + if err != nil { + return indices, err + } + indices = append(indices, fileDefault) + return indices, nil +} + +func (i *Index) init() error { + if i.Version == "" || i.IndexName == "" || i.BeatDir == "" || i.BeatName == "" { + return errors.New("RequiredParams: Version, IndexName, BeatDir and BeatName") + } + i.BeatName = clean(i.BeatName) + + i.fieldsYaml = filepath.Join(i.BeatDir, "fields.yml") + if _, err := os.Stat(i.fieldsYaml); err != nil { + return err + } + + i.targetDirDefault = createTargetDir(i.BeatDir, "default") + i.targetDir5x = createTargetDir(i.BeatDir, "5.x") + i.targetFilename = i.BeatName + ".json" + + return nil +} + +func clean(name string) string { + reg := regexp.MustCompile("[^a-zA-Z0-9_]+") + return reg.ReplaceAllString(name, "") +} + +func dumpToFile(f string, pattern common.MapStr) error { + patternIndent, err := json.MarshalIndent(pattern, "", " ") + if err != nil { + return err + } + err = ioutil.WriteFile(f, patternIndent, 0644) + if err != nil { + return err + } + return nil +} + +func createTargetDir(baseDir string, version string) string { + targetDir := filepath.Join(baseDir, "_meta", "kibana", version, "index-pattern") + if _, err := os.Stat(targetDir); os.IsNotExist(err) { + os.MkdirAll(targetDir, 0777) + } + return targetDir +} diff --git a/libbeat/kibana/index_test.go b/libbeat/kibana/index_test.go new file mode 100644 index 00000000000..faa5b94ba6d --- /dev/null +++ b/libbeat/kibana/index_test.go @@ -0,0 +1,174 @@ +package kibana + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +func TestAllArgsSet(t *testing.T) { + beatDir := tmpPath() + defer teardown(beatDir) + + tests := []struct { + Index Index + }{ + {Index: Index{IndexName: "beat-index", BeatDir: beatDir, BeatName: "mybeat."}}, + {Index: Index{Version: "6.0", BeatDir: beatDir, BeatName: "mybeat."}}, + {Index: Index{Version: "6.0", IndexName: "beat-index", BeatName: "mybeat."}}, + {Index: Index{Version: "6.0", IndexName: "beat-index", BeatDir: beatDir}}, + } + for idx, test := range tests { + err := test.Index.init() + msg := fmt.Sprintf("(%v): Should have raised error", idx) + assert.Error(t, err, msg) + } +} + +func TestInit(t *testing.T) { + beatDir := tmpPath() + defer teardown(beatDir) + // checks for fields.yml + idx := Index{Version: "7.0", IndexName: "beat-index", BeatDir: filepath.Join(beatDir, "notexistent"), BeatName: "mybeat."} + err := idx.init() + assert.Error(t, err) + + idx = Index{Version: "7.0", IndexName: "beat-index", BeatDir: beatDir, BeatName: "mybeat."} + err = idx.init() + assert.NoError(t, err) + + // creates file dir and sets name + expectedDir := filepath.Join(beatDir, "_meta/kibana/default/index-pattern") + assert.Equal(t, expectedDir, idx.targetDirDefault) + _, err = os.Stat(idx.targetDirDefault) + assert.NoError(t, err) + + expectedDir = filepath.Join(beatDir, "_meta/kibana/5.x/index-pattern") + assert.Equal(t, expectedDir, idx.targetDir5x) + _, err = os.Stat(idx.targetDir5x) + assert.NoError(t, err) + + assert.Equal(t, "mybeat.json", idx.targetFilename) +} + +func TestCleanName(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {input: " beat index pattern", expected: "beatindexpattern"}, + {input: "Beat@Index.!", expected: "BeatIndex"}, + {input: "beatIndex", expected: "beatIndex"}, + } + for idx, test := range tests { + output := clean(test.input) + msg := fmt.Sprintf("(%v): Expected <%s> Received: <%s>", idx, test.expected, output) + assert.Equal(t, test.expected, output, msg) + } +} + +func TestDefault(t *testing.T) { + beatDir := tmpPath() + defer teardown(beatDir) + + index := Index{Version: "7.0.0-alpha1", IndexName: "metricbeat-*", BeatDir: beatDir, BeatName: "metric beat !"} + index.Create() + + tests := []map[string]string{ + {"existing": "metricbeat-5x-old.json", "created": "_meta/kibana/5.x/index-pattern/metricbeat.json"}, + {"existing": "metricbeat-default-old.json", "created": "_meta/kibana/default/index-pattern/metricbeat.json"}, + } + + for _, test := range tests { + // compare default + existing, err := readJson(filepath.Join(beatDir, test["existing"])) + assert.NoError(t, err) + created, err := readJson(filepath.Join(beatDir, test["created"])) + assert.NoError(t, err) + + var attrExisting, attrCreated common.MapStr + + if strings.Contains(test["existing"], "default") { + assert.Equal(t, existing["version"], created["version"]) + + objExisting := existing["objects"].([]interface{})[0].(map[string]interface{}) + objCreated := created["objects"].([]interface{})[0].(map[string]interface{}) + + assert.Equal(t, objExisting["version"], objCreated["version"]) + assert.Equal(t, objExisting["id"], objCreated["id"]) + assert.Equal(t, objExisting["type"], objCreated["type"]) + + attrExisting = objExisting["attributes"].(map[string]interface{}) + attrCreated = objCreated["attributes"].(map[string]interface{}) + } else { + attrExisting = existing + attrCreated = created + } + + // check fieldFormatMap + var ffmExisting, ffmCreated map[string]interface{} + err = json.Unmarshal([]byte(attrExisting["fieldFormatMap"].(string)), &ffmExisting) + assert.NoError(t, err) + err = json.Unmarshal([]byte(attrCreated["fieldFormatMap"].(string)), &ffmCreated) + assert.NoError(t, err) + assert.Equal(t, ffmExisting, ffmCreated) + + // check fields + var fieldsExisting, fieldsCreated []map[string]interface{} + err = json.Unmarshal([]byte(attrExisting["fields"].(string)), &fieldsExisting) + assert.NoError(t, err) + err = json.Unmarshal([]byte(attrCreated["fields"].(string)), &fieldsCreated) + assert.NoError(t, err) + assert.Equal(t, len(ffmExisting), len(ffmCreated)) + for _, e := range fieldsExisting { + idx := find(fieldsCreated, e["name"].(string)) + assert.NotEqual(t, -1, idx) + assert.Equal(t, fieldsCreated[idx], e) + } + } +} + +func find(a []map[string]interface{}, k string) int { + for idx, e := range a { + if e["name"].(string) == k { + return idx + } + } + return -1 +} + +func readJson(path string) (map[string]interface{}, error) { + f, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + var data map[string]interface{} + err = json.Unmarshal(f, &data) + if err != nil { + return nil, err + } + return data, nil +} + +func tmpPath() string { + beatDir, err := filepath.Abs("./testdata") + if err != nil { + panic(err) + } + return beatDir +} + +func teardown(path string) { + if path == "" { + path = tmpPath() + } + os.RemoveAll(filepath.Join(path, "_meta")) +} diff --git a/libbeat/kibana/testdata/fields.yml b/libbeat/kibana/testdata/fields.yml new file mode 100644 index 00000000000..f8a8c7d835e --- /dev/null +++ b/libbeat/kibana/testdata/fields.yml @@ -0,0 +1,6858 @@ + +- key: beat + title: Beat + description: > + Contains common beat fields available in all event types. + fields: + + - name: beat.name + description: > + The name of the Beat sending the log messages. If the Beat name is + set in the configuration file, then that value is used. If it is not + set, the hostname is used. To set the Beat name, use the `name` + option in the configuration file. + - name: beat.hostname + description: > + The hostname as returned by the operating system on which the Beat is + running. + - name: beat.timezone + description: > + The timezone as returned by the operating system on which the Beat is + running. + - name: beat.version + description: > + The version of the beat that generated this event. + + - name: "@timestamp" + type: date + required: true + format: date + example: August 26th 2016, 12:35:53.332 + description: > + The timestamp when the event log record was generated. + + - name: tags + description: > + Arbitrary tags that can be set per Beat and per transaction + type. + + - name: fields + type: object + object_type: keyword + description: > + Contains user configurable fields. + + - name: error + type: group + description: > + Error fields containing additional info in case of errors. + fields: + - name: message + type: text + description: > + Error message. + - name: code + type: long + description: > + Error code. + - name: type + type: keyword + description: > + Error type. +- key: cloud + title: Cloud provider metadata + description: > + Metadata from cloud providers added by the add_cloud_metadata processor. + fields: + + - name: meta.cloud.provider + example: ec2 + description: > + Name of the cloud provider. Possible values are ec2, gce, or digitalocean. + + - name: meta.cloud.instance_id + description: > + Instance ID of the host machine. + + - name: meta.cloud.instance_name + description: > + Instance name of the host machine. + + - name: meta.cloud.machine_type + example: t2.medium + description: > + Machine type of the host machine. + + - name: meta.cloud.availability_zone + example: us-east-1c + description: > + Availability zone in which this host is running. + + - name: meta.cloud.project_id + example: project-x + description: > + Name of the project in Google Cloud. + + - name: meta.cloud.region + description: > + Region in which this host is running. +- key: docker + title: Docker + description: > + beta[] + + Docker stats collected from Docker. + short_config: false + anchor: docker-processor + fields: + - name: docker + type: group + fields: + - name: container.id + type: keyword + description: > + Unique container id. + - name: container.image + type: keyword + description: > + Name of the image the container was built on. + - name: container.name + type: keyword + description: > + Container name. + - name: container.labels + type: object + object_type: keyword + description: > + Image labels. +- key: kubernetes + title: Kubernetes + description: > + beta[] + + Kubernetes metadata added by the kubernetes processor + short_config: false + anchor: kubernetes-processor + fields: + - name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: labels + type: object + description: > + Kubernetes labels map + + - name: annotations + type: object + description: > + Kubernetes annotations map + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: > + Kubernetes container image +- key: common + title: Common + description: > + Contains common fields available in all event types. + fields: + + - name: metricset.module + description: > + The name of the module that generated the event. + + - name: metricset.name + description: > + The name of the metricset that generated the event. + + - name: metricset.host + description: > + Hostname of the machine from which the metricset was collected. This + field may not be present when the data was collected locally. + + - name: metricset.rtt + type: long + required: true + description: > + Event round trip time in microseconds. + + - name: metricset.namespace + type: keyword + description: > + Namespace of dynamic metricsets. + + - name: type + required: true + example: metricsets + description: > + The document type. Always set to "metricsets". + +- key: aerospike + title: "Aerospike" + description: > + experimental[] + + Aerospike module + fields: + - name: aerospike + type: group + description: > + fields: + - name: namespace + type: group + description: > + namespace + fields: + - name: client + type: group + description: > + Client stats. + fields: + - name: delete + type: group + description: > + Client delete transactions stats. + fields: + - name: error + type: long + description: > + Number of client delete transactions that failed with an error. + - name: not_found + type: long + description: > + Number of client delete transactions that resulted in a not found. + - name: success + type: long + description: > + Number of successful client delete transactions. + - name: timeout + type: long + description: > + Number of client delete transactions that timed out. + - name: read + type: group + description: > + Client read transactions stats. + fields: + - name: error + type: long + description: > + Number of client read transaction errors. + - name: not_found + type: long + description: > + Number of client read transaction that resulted in not found. + - name: success + type: long + description: > + Number of successful client read transactions. + - name: timeout + type: long + description: > + Number of client read transaction that timed out. + - name: write + type: group + description: > + Client write transactions stats. + fields: + - name: error + type: long + description: > + Number of client write transactions that failed with an error. + - name: success + type: long + description: > + Number of successful client write transactions. + - name: timeout + type: long + description: > + Number of client write transactions that timed out. + - name: device + type: group + description: > + Disk storage stats + fields: + - name: available.pct + type: scaled_float + format: percent + description: > + Measures the minimum contiguous disk space across all disks in a namespace. + - name: free.pct + type: scaled_float + format: percent + description: > + Percentage of disk capacity free for this namespace. + - name: total.bytes + type: long + format: bytes + description: > + Total bytes of disk space allocated to this namespace on this node. + - name: used.bytes + type: long + format: bytes + description: > + Total bytes of disk space used by this namespace on this node. + - name: hwm_breached + type: boolean + description: > + If true, Aerospike has breached 'high-water-[disk|memory]-pct' for this namespace. + - name: memory + type: group + description: > + Memory storage stats. + fields: + - name: free.pct + type: scaled_float + format: percent + description: > + Percentage of memory capacity free for this namespace on this node. + - name: used.data.bytes + type: long + format: bytes + description: > + Amount of memory occupied by data for this namespace on this node. + - name: used.index.bytes + type: long + format: bytes + description: > + Amount of memory occupied by the index for this namespace on this node. + - name: used.sindex.bytes + type: long + format: bytes + description: > + Amount of memory occupied by secondary indexes for this namespace on this node. + - name: used.total.bytes + type: long + format: bytes + description: > + Total bytes of memory used by this namespace on this node. + - name: name + type: keyword + description: > + Namespace name + - name: node.host + type: keyword + desription: > + Node host + - name: node.name + type: keyword + description: > + Node name + - name: objects + type: group + description: > + Records stats. + fields: + - name: master + type: long + description: > + Number of records on this node which are active masters. + - name: total + type: long + description: > + Number of records in this namespace for this node. + - name: stop_writes + type: boolean + description: > + If true this namespace is currently not allowing writes. + +- key: apache + title: "Apache" + description: > + Apache HTTPD server metricsets collected from the Apache web server. + short_config: false + fields: + - name: apache + type: group + description: > + `apache` contains the metrics that were scraped from Apache. + fields: + - name: status + type: group + description: > + `status` contains the metrics that were scraped from the Apache status page. + fields: + - name: hostname + type: keyword + description: > + Apache hostname. + - name: total_accesses + type: long + description: > + Total number of access requests. + - name: total_kbytes + type: long + description: > + Total number of kilobytes served. + - name: requests_per_sec + type: scaled_float + description: > + Requests per second. + - name: bytes_per_sec + type: scaled_float + description: > + Bytes per second. + - name: bytes_per_request + type: scaled_float + description: > + Bytes per request. + - name: workers.busy + type: long + description: > + Number of busy workers. + - name: workers.idle + type: long + description: > + Number of idle workers. + - name: uptime + type: group + description: > + Uptime stats. + fields: + - name: server_uptime + type: long + description: > + Server uptime in seconds. + - name: uptime + type: long + description: > + Server uptime. + - name: cpu + type: group + description: > + CPU stats. + fields: + - name: load + type: scaled_float + description: > + CPU Load. + - name: user + type: scaled_float + description: > + CPU user load. + - name: system + type: scaled_float + description: > + System cpu. + - name: children_user + type: scaled_float + description: > + CPU of children user. + - name: children_system + type: scaled_float + description: > + CPU of children system. + - name: connections + type: group + description: > + Connection stats. + fields: + - name: total + type: long + description: > + Total connections. + - name: async.writing + type: long + description: > + Async connection writing. + - name: async.keep_alive + type: long + description: > + Async keeped alive connections. + - name: async.closing + type: long + description: > + Async closed connections. + - name: load + type: group + description: > + Load averages. + fields: + - name: "1" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last minute. + - name: "5" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 5 minutes. + - name: "15" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 15 minutes. + - name: scoreboard + type: group + description: > + Scoreboard metrics. + fields: + - name: starting_up + type: long + description: > + Starting up. + - name: reading_request + type: long + description: > + Reading requests. + - name: sending_reply + type: long + description: > + Sending Reply. + - name: keepalive + type: long + description: > + Keep alive. + - name: dns_lookup + type: long + description: > + Dns Lookups. + - name: closing_connection + type: long + description: > + Closing connections. + - name: logging + type: long + description: > + Logging + - name: gracefully_finishing + type: long + description: > + Gracefully finishing. + - name: idle_cleanup + type: long + description: > + Idle cleanups. + - name: open_slot + type: long + description: > + Open slots. + - name: waiting_for_connection + type: long + description: > + Waiting for connections. + - name: total + type: long + description: > + Total. + +- key: ceph + title: "Ceph" + description: > + beta[] + + Ceph module + short_config: false + fields: + - name: ceph + type: group + description: > + `ceph` contains the metrics that were scraped from CEPH. + fields: + - name: cluster_disk + type: group + description: > + cluster_disk + fields: + - name: available.bytes + type: long + description: > + Available bytes of the cluster + format: bytes + - name: total.bytes + type: long + description: > + Total bytes of the cluster + format: bytes + - name: used.bytes + type: long + description: > + Used bytes of the cluster + format: bytes + + - name: cluster_health + type: group + description: > + cluster_health + fields: + - name: overall_status + type: keyword + description: > + Overall status of the cluster + - name: timechecks.epoch + type: long + description: > + Map version + - name: timechecks.round.value + type: long + description: > + timecheck round + - name: timechecks.round.status + type: keyword + description: > + Status of the round + + - name: cluster_status + type: group + description: > + cluster_status + fields: + - name: version + type: long + description: > + Ceph Status version + - name: traffic.read_bytes + type: long + format: bytes + description: > + Cluster read throughput per second + - name: traffic.write_bytes + type: long + format: bytes + description: > + Cluster write throughput per second + - name: traffic.read_op_per_sec + type: long + description: > + Cluster read iops per second + - name: traffic.write_op_per_sec + type: long + description: > + Cluster write iops per second + - name: misplace.total + type: long + description: > + Cluster misplace pg number + - name: misplace.objects + type: long + description: > + Cluster misplace objects number + - name: misplace.ratio + type: scaled_float + format: percent + description: > + Cluster misplace ratio + - name: degraded.total + type: long + description: > + Cluster degraded pg number + - name: degraded.objects + type: long + description: > + Cluster degraded objects number + - name: degraded.ratio + type: scaled_float + format: percent + description: > + Cluster degraded ratio + - name: pg.data_bytes + type: long + format: bytes + description: > + Cluster pg data bytes + - name: pg.avail_bytes + type: long + format: bytes + description: > + Cluster available bytes + - name: pg.total_bytes + type: long + format: bytes + description: > + Cluster total bytes + - name: pg.used_bytes + type: long + format: bytes + description: > + Cluster used bytes + - name: pg_state.state_name + type: long + description: > + Pg state description + - name: pg_state.count + type: long + description: > + Shows how many pgs are in state of pg_state.state_name + - name: pg_state.version + type: long + description: > + Cluster status version + - name: osd.full + type: boolean + description: > + Is osd full + - name: osd.nearfull + type: boolean + description: > + Is osd near full + - name: osd.num_osds + type: long + description: > + Shows how many osds in the cluster + - name: osd.num_up_osds + type: long + description: > + Shows how many osds are on the state of UP + - name: osd.num_in_osds + type: long + description: > + Shows how many osds are on the state of IN + - name: osd.num_in_osds2 + type: long + description: > + Shows how many osds are on the state of REMAPPED + - name: osd.epoch + type: long + description: > + epoch number + + - name: monitor_health + type: group + description: > + monitor_health stats data + fields: + - name: available.pct + type: long + description: > + Available percent of the MON + - name: health + type: keyword + description: > + Health of the MON + - name: available.kb + type: long + description: > + Available KB of the MON + - name: total.kb + type: long + description: > + Total KB of the MON + - name: used.kb + type: long + description: > + Used KB of the MON + - name: last_updated + type: date + description: > + Time when was updated + - name: name + type: keyword + description: > + Name of the MON + - name: store_stats.log.bytes + type: long + description: > + Log bytes of MON + format: bytes + - name: store_stats.misc.bytes + type: long + description: > + Misc bytes of MON + format: bytes + - name: store_stats.sst.bytes + type: long + description: > + SST bytes of MON + format: bytes + - name: store_stats.total.bytes + type: long + description: > + Total bytes of MON + format: bytes + - name: store_stats.last_updated + type: long + description: > + Last updated + + - name: pool_disk + type: group + description: > + pool_disk + fields: + - name: id + type: long + description: > + Id of the pool + - name: name + type: keyword + description: > + Name of the pool + - name: stats.available.bytes + type: long + description: > + Available bytes of the pool + format: bytes + - name: stats.objects + type: long + description: > + Number of objects of the pool + - name: stats.used.bytes + type: long + description: > + Used bytes of the pool + format: bytes + - name: stats.used.kb + type: long + description: > + Used kb of the pool + +- key: couchbase + title: "Couchbase" + description: > + beta[] + + Metrics collected from Couchbase servers. + short_config: false + fields: + - name: couchbase + type: group + description: > + `couchbase` contains the metrics that were scraped from Couchbase. + fields: + - name: bucket + type: group + description: > + Couchbase bucket metrics. + fields: + - name: name + type: keyword + description: > + Name of the bucket. + - name: type + type: keyword + description: > + Type of the bucket. + - name: data.used.bytes + format: bytes + type: long + description: > + Size of user data within buckets of the specified state that are resident in RAM. + - name: disk.fetches + type: long + description: > + Number of disk fetches. + - name: disk.used.bytes + format: bytes + type: long + description: > + Amount of disk used (bytes). + - name: memory.used.bytes + format: bytes + type: long + description: > + Amount of memory used by the bucket (bytes). + - name: quota.ram.bytes + format: bytes + type: long + description: > + Amount of RAM used by the bucket (bytes). + - name: quota.use.pct + format: percent + type: scaled_float + description: > + Percentage of RAM used (for active objects) against the configured bucket size (%). + - name: ops_per_sec + type: long + description: > + Number of operations per second. + - name: item_count + type: long + description: > + Number of items associated with the bucket. + + - name: cluster + type: group + description: > + Couchbase cluster metrics. + fields: + - name: hdd.free.bytes + format: bytes + type: long + description: > + Free hard drive space in the cluster (bytes). + - name: hdd.quota.total.bytes + format: bytes + type: long + description: > + Hard drive quota total for the cluster (bytes). + - name: hdd.total.bytes + format: bytes + type: long + description: > + Total hard drive space available to the cluster (bytes). + - name: hdd.used.value.bytes + format: bytes + type: long + description: > + Hard drive space used by the cluster (bytes). + - name: hdd.used.by_data.bytes + format: bytes + type: long + description: > + Hard drive space used by the data in the cluster (bytes). + - name: max_bucket_count + type: long + description: > + Max bucket count setting. + - name: quota.index_memory.mb + type: long + description: > + Memory quota setting for the Index service (Mbyte). + - name: quota.memory.mb + type: long + description: > + Memory quota setting for the cluster (Mbyte). + - name: ram.quota.total.value.bytes + format: bytes + type: long + description: > + RAM quota total for the cluster (bytes). + - name: ram.quota.total.per_node.bytes + format: bytes + type: long + description: > + RAM quota used by the current node in the cluster (bytes). + - name: ram.quota.used.value.bytes + format: bytes + type: long + description: > + RAM quota used by the cluster (bytes). + - name: ram.quota.used.per_node.bytes + format: bytes + type: long + description: > + Ram quota used by the current node in the cluster (bytes) + - name: ram.total.bytes + format: bytes + type: long + description: > + Total RAM available to cluster (bytes). + - name: ram.used.value.bytes + format: bytes + type: long + description: > + RAM used by the cluster (bytes). + - name: ram.used.by_data.bytes + format: bytes + type: long + description: > + RAM used by the data in the cluster (bytes). + + - name: node + type: group + description: > + Couchbase node metrics. + fields: + - name: cmd_get + type: long + description: > + Number of get commands + - name: couch.docs.disk_size.bytes + format: bytes + type: long + description: > + Amount of disk space used by Couch docs (bytes). + - name: couch.docs.data_size.bytes + format: bytes + type: long + description: > + Data size of Couch docs associated with a node (bytes). + - name: couch.spatial.data_size.bytes + type: long + description: > + Size of object data for spatial views (bytes). + - name: couch.spatial.disk_size.bytes + type: long + description: > + Amount of disk space used by spatial views (bytes). + - name: couch.views.disk_size.bytes + type: long + description: > + Amount of disk space used by Couch views (bytes). + - name: couch.views.data_size.bytes + type: long + description: > + Size of object data for Couch views (bytes). + - name: cpu_utilization_rate.pct + type: scaled_float + description: > + The CPU utilization rate (%). + - name: current_items.value + type: long + description: > + Number of current items. + - name: current_items.total + type: long + description: > + Total number of items associated with the node. + - name: ep_bg_fetched + type: long + description: > + Number of disk fetches performed since the server was started. + - name: get_hits + type: long + description: > + Number of get hits. + - name: hostname + type: keyword + description: > + The hostname of the node. + - name: mcd_memory.allocated.bytes + format: bytes + type: long + description: > + Amount of memcached memory allocated (bytes). + - name: mcd_memory.reserved.bytes + type: long + description: > + Amount of memcached memory reserved (bytes). + - name: memory.free.bytes + type: long + description: > + Amount of memory free for the node (bytes). + - name: memory.total.bytes + type: long + description: > + Total memory available to the node (bytes). + - name: memory.used.bytes + type: long + description: > + Memory used by the node (bytes). + - name: ops + type: long + description: > + Number of operations performed on Couchbase. + - name: swap.total.bytes + type: long + description: > + Total swap size allocated (bytes). + - name: swap.used.bytes + type: long + description: > + Amount of swap space used (bytes). + - name: uptime.sec + type: long + description: > + Time during which the node was in operation (sec). + - name: vb_replica_curr_items + type: long + description: > + Number of items/documents that are replicas. + +- key: docker + title: "Docker" + description: > + beta[] + + Docker stats collected from Docker. + short_config: false + fields: + - name: docker + type: group + description: > + Information and statistics about docker's running containers. + fields: + - name: container + type: group + description: > + Docker container metrics. + fields: + - name: command + type: keyword + description: > + Command that was executed in the Docker container. + - name: created + type: date + description: > + Date when the container was created. + - name: status + type: keyword + description: > + Container status. + - name: size + type: group + description: > + Container size metrics. + fields: + - name: root_fs + type: long + description: > + Total size of all the files in the container. + - name: rw + type: long + description: > + Size of the files that have been created or changed since creation. + - name: tags + type: array + description: > + Image tags. + + - name: cpu + type: group + description: > + Runtime CPU metrics. + fields: + - name: kernel.pct + type: scaled_float + format: percentage + description: > + The system kernel consumed by the Docker server. + - name: kernel.ticks + type: long + description: > + CPU kernel ticks. + - name: system.pct + type: scaled_float + format: percentage + description: > + - name: system.ticks + type: long + description: > + CPU system ticks. + - name: user.pct + type: scaled_float + format: percentage + description: > + - name: user.ticks + type: long + description: > + CPU user ticks + - name: total.pct + type: scaled_float + format: percentage + description: > + Total CPU usage. + # TODO: how to document cpu list? + #- name: core + # type: array + # description: > + # Dictionary with list of cpu and usage inside. + + - name: diskio + type: group + description: > + Disk I/O metrics. + fields: + - name: reads + type: scaled_float + description: > + Number of reads. + - name: writes + type: scaled_float + description: > + Number of writes. + - name: total + type: scaled_float + description: > + Number of reads and writes combined. + + - name: healthcheck + type: group + description: > + Docker container metrics. + fields: + - name: failingstreak + type: integer + description: > + concurent failed check + - name: status + type: keyword + description: > + Healthcheck status code + - name: event + type: group + description: > + event fields. + fields: + - name: end_date + type: date + description: > + Healthcheck end date + - name: start_date + type: date + description: > + Healthcheck start date + - name: output + type: keyword + description: > + Healthcheck output + - name: exit_code + type: integer + description: > + Healthcheck status code + + - name: image + type: group + description: > + Docker image metrics. + fields: + - name: id + type: group + description: > + The image layers identifier. + fields: + - name: current + type: keyword + description: > + Unique image identifier given upon its creation. + - name: parent + type: keyword + description: > + Identifier of the image, if it exists, from which the current image directly descends. + - name: created + type: date + description: > + Date and time when the image was created. + - name: size + type: group + description: > + Image size layers. + fields: + - name: virtual + type: long + description: > + Size of the image. + - name: regular + type: long + description: > + Total size of the all cached images associated to the current image. + + - name: labels + type: object + object_type: keyword + description: > + Image labels. + + - name: tags + type: array + description: > + Image tags. + + - name: info + type: group + description: > + beta[] + + Info metrics based on https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/display-system-wide-information. + fields: + - name: containers + type: group + description: > + Overall container stats. + fields: + - name: paused + type: long + description: > + Total number of paused containers. + - name: running + type: long + description: > + Total number of running containers. + - name: stopped + type: long + description: > + Total number of stopped containers. + - name: total + type: long + description: > + Total number of existing containers. + - name: id + type: keyword + description: > + Unique Docker host identifier. + + - name: images + type: long + description: > + Total number of existing images. + + - name: memory + type: group + description: > + Memory metrics. + fields: + + - name: fail.count + type: scaled_float + description: > + Fail counter. + - name: limit + type: long + format: bytes + description: > + Memory limit. + - name: rss + type: group + description: > + RSS memory stats. + fields: + - name: total + type: long + format: bytes + description: > + Total memory resident set size. + - name: pct + type: scaled_float + format: percentage + description: > + Memory resident set size percentage. + - name: usage + type: group + description: > + Usage memory stats. + fields: + - name: max + type: long + format: bytes + description: > + Max memory usage. + - name: pct + type: scaled_float + format: percentage + description: > + Memory usage percentage. + - name: total + type: long + format: bytes + description: > + Total memory usage. + + - name: network + type: group + description: > + Network metrics. + fields: + + - name: interface + type: keyword + description: > + Network interface name. + - name: in + type: group + description: > + Incoming network stats. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of incoming bytes. + - name: dropped + type: scaled_float + description: > + Total number of dropped incoming packets. + - name: errors + type: long + description: > + Total errors on incoming packets. + - name: packets + type: long + description: > + Total number of incoming packets. + - name: out + type: group + description: > + Outgoing network stats. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of outgoing bytes. + - name: dropped + type: scaled_float + description: > + Total number of dropped outgoing packets. + - name: errors + type: long + description: > + Total errors on outgoing packets. + - name: packets + type: long + description: > + Total number of outgoing packets. + +- key: dropwizard + title: "Dropwizard" + description: > + beta[] + + Stats collected from Dropwizard. + short_config: false + fields: + - name: dropwizard + type: group + description: > + fields: + +- key: elasticsearch + title: "Elasticsearch" + description: > + experimental[] + + Elasticsearch module + short_config: false + fields: + - name: elasticsearch + type: group + description: > + fields: + - name: cluster.name + type: keyword + description: > + Elasticsearch cluster name. + - name: node + type: group + description: > + node + fields: + - name: jvm.memory.heap_init.bytes + type: long + format: bytes + description: > + Heap init used by the JVM in bytes. + - name: jvm.version + type: keyword + description: > + JVM version. + - name: name + type: keyword + description: > + Node name. + - name: version + type: keyword + description: > + Node version. + + - name: node.stats + type: group + description: > + node_stats + fields: + - name: indices + type: group + description: > + Node indices stats + fields: + - name: docs.count + type: long + description: > + Total number of existing documents. + - name: docs.deleted + type: long + description: > + Total number of deleted documents. + - name: segments.count + type: long + description: > + Total number of segments. + - name: segments.memory.bytes + type: long + format: bytes + description: > + Total size of segments in bytes. + - name: store.size.bytes + type: long + description: > + Total size of the store in bytes. + - name: jvm.mem.pools + type: group + description: > + JVM memory pool stats + fields: + - name: old + type: group + description: > + Old memory pool stats. + fields: + - name: max.bytes + type: long + format: bytes + description: + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: + Used bytes. + - name: young + type: group + description: > + Young memory pool stats. + fields: + - name: max.bytes + type: long + format: bytes + description: + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: + Used bytes. + - name: survivor + type: group + description: > + Survivor memory pool stats. + fields: + - name: max.bytes + type: long + format: bytes + description: + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: + Used bytes. + + - name: jvm.gc.collectors + type: group + description: > + GC collector stats. + fields: + - name: old.collection + type: group + description: > + Old collection gc. + fields: + - name: count + type: long + description: > + - name: ms + type: long + description: > + - name: young.collection + type: group + description: > + Young collection gc. + fields: + - name: count + type: long + description: > + - name: ms + type: long + description: > + +- key: golang + title: "Golang" + description: > + Golang module + short_config: false + fields: + - name: golang + type: group + description: > + fields: + - name: expvar + type: group + description: > + expvar + fields: + - name: cmdline + type: keyword + description: > + The cmdline of this golang program start with. + - name: heap + type: group + description: > + The golang program heap information exposed by expvar. + fields: + - name: cmdline + type: keyword + description: > + The cmdline of this golang program start with. + + - name: gc + type: group + description: > + Garbage collector summary. + fields: + - name: total_pause + type: group + description: > + Total GC pause duration over lifetime of process. + fields: + - name: ns + type: long + description: > + Duration in Ns. + - name: total_count + type: long + description: > + Total number of GC was happened. + - name: next_gc_limit + type: long + format: bytes + description: > + Next collection will happen when HeapAlloc > this amount. + - name: cpu_fraction + type: long + description: > + Fraction of CPU time used by GC. + - name: pause + type: group + description: > + Last GC pause durations during the monitoring period. + fields: + - name: count + type: long + description: > + Count of GC pause duration during this collect period. + - name: sum + type: group + description: > + Total GC pause duration during this collect period. + fields: + - name: ns + type: long + description: > + Duration in Ns. + - name: max + type: group + description: > + Max GC pause duration during this collect period. + fields: + - name: ns + type: long + description: > + Duration in Ns. + - name: avg + type: group + description: > + Average GC pause duration during this collect period. + fields: + - name: ns + type: long + description: > + Duration in Ns. + + - name: system + type: group + description: > + Heap summary,which bytes was obtained from system. + fields: + - name: total + type: long + format: bytes + description: > + Total bytes obtained from system (sum of XxxSys below). + - name: optained + type: long + format: bytes + description: > + Via HeapSys, bytes obtained from system. heap_sys = heap_idle + heap_inuse. + - name: stack + type: long + format: bytes + description: > + Bytes used by stack allocator, and these bytes was obtained from system. + - name: released + type: long + format: bytes + description: > + Bytes released to the OS. + + - name: allocations + type: group + description: > + Heap allocations summary. + fields: + - name: mallocs + type: long + description: > + Number of mallocs. + - name: frees + type: long + description: > + Number of frees. + - name: objects + type: long + description: > + Total number of allocated objects. + - name: total + type: long + format: bytes + description: > + Bytes allocated (even if freed) throughout the lifetime. + - name: allocated + type: long + format: bytes + description: > + Bytes allocated and not yet freed (same as Alloc above). + - name: idle + type: long + format: bytes + description: > + Bytes in idle spans. + - name: active + type: long + format: bytes + description: > + Bytes in non-idle span. +- key: graphite + title: "graphite" + description: > + []experimental + + graphite Module + fields: + - name: graphite + type: group + description: > + fields: + - name: server + type: group + description: > + server + fields: + - name: example + type: keyword + description: > + Example field + +- key: haproxy + title: "HAProxy" + description: > + HAProxy Module + short_config: false + fields: + - name: haproxy + type: group + description: > + HAProxy metrics. + fields: + - name: info + type: group + description: > + General information about HAProxy processes. + fields: + - name: processes + type: long + description: > + Number of processes. + + - name: process_num + type: long + description: > + Process number. + + - name: pid + type: long + description: > + Process ID. + + - name: run_queue + type: long + description: > + + - name: tasks + type: long + description: > + + - name: uptime.sec + type: long + description: > + Current uptime in seconds. + + - name: memory.max.bytes + type: long + format: bytes + description: > + Maximum amount of memory usage in bytes (the 'Memmax_MB' value converted to bytes). + + - name: ulimit_n + type: long + description: > + Maximum number of open files for the process. + + - name: compress + type: group + description: > + + fields: + - name: bps + type: group + description: > + + fields: + - name: in + type: long + description: > + + - name: out + type: long + description: > + + - name: rate_limit + type: long + description: > + + - name: connection + type: group + description: > + + fields: + - name: rate + type: group + description: > + + fields: + - name: value + type: long + description: > + + - name: limit + type: long + description: > + + - name: max + type: long + description: > + + - name: current + type: long + description: > + Current connections. + + - name: total + type: long + description: > + Total connections. + + - name: ssl.current + type: long + description: > + Current SSL connections. + + - name: ssl.total + type: long + description: > + Total SSL connections. + + - name: ssl.max + type: long + description: > + Maximum SSL connections. + + - name: max + type: long + description: > + Maximum connections. + + - name: hard_max + type: long + description: > + + - name: requests.total + type: long + description: > + + - name: sockets.max + type: long + description: > + + - name: requests.max + type: long + description: > + + - name: pipes + type: group + description: > + fields: + - name: used + type: integer + description: > + + - name: free + type: integer + description: > + + - name: max + type: integer + description: > + + - name: session + type: group + description: + fields: + - name: rate.value + type: integer + description: > + + - name: rate.limit + type: integer + description: > + + - name: rate.max + type: integer + description: > + + + - name: ssl + type: group + description: + fields: + - name: rate.value + type: integer + description: + + - name: rate.limit + type: integer + description: + + - name: rate.max + type: integer + description: + + - name: frontend + type: group + description: + fields: + - name: key_rate.value + type: integer + description: + + - name: key_rate.max + type: integer + description: + + - name: session_reuse.pct + type: scaled_float + format: percent + description: + + - name: backend + type: group + description: + fields: + - name: key_rate.value + type: integer + description: + + - name: key_rate.max + type: integer + description: MaxConnRate + - name: cached_lookups + type: long + description: + - name: cache_misses + type: long + description: + + + - name: zlib_mem_usage + type: group + description: > + + fields: + - name: value + type: integer + description: > + + - name: max + type: integer + description: > + + - name: idle.pct + type: scaled_float + format: percent + description: > + + - name: stat + type: group + description: > + Stats collected from HAProxy processes. + fields: + + - name: status + type: keyword + description: > + Status (UP, DOWN, NOLB, MAINT, or MAINT(via)...). + + - name: weight + type: long + description: > + Total weight (for backends), or server weight (for servers). + + - name: downtime + type: long + description: > + Total downtime (in seconds). For backends, this value is the downtime + for the whole backend, not the sum of the downtime for the servers. + + - name: component_type + type: integer + description: > + Component type (0=frontend, 1=backend, 2=server, or 3=socket/listener). + + - name: process_id + type: integer + description: > + Process ID (0 for first instance, 1 for second, and so on). + + - name: service_name + type: keyword + description: > + Service name (FRONTEND for frontend, BACKEND for backend, or any name for server/listener). + + - name: in.bytes + type: long + format: bytes + description: > + Bytes in. + + - name: out.bytes + type: long + format: bytes + description: > + Bytes out. + + - name: last_change + type: integer + description: > + Number of seconds since the last UP->DOWN or DOWN->UP transition. + + - name: throttle.pct + type: scaled_float + format: percentage + description: > + Current throttle percentage for the server when slowstart + is active, or no value if slowstart is inactive. + + - name: selected.total + type: long + description: > + Total number of times a server was selected, either for new + sessions, or when re-dispatching. For servers, this field reports the + the number of times the server was selected. + + - name: tracked.id + type: long + description: > + ID of the proxy/server if tracking is enabled. + + - name: connection + type: group + fields: + + - name: total + type: long + description: > + Cumulative number of connections. + + - name: retried + type: long + description: > + Number of times a connection to a server was retried. + + - name: time.avg + type: long + description: > + Average connect time in ms over the last 1024 requests. + + - name: request + type: group + fields: + + - name: denied + type: long + description: > + Requests denied because of security concerns. + + * For TCP this is because of a matched tcp-request content rule. + * For HTTP this is because of a matched http-request or tarpit rule. + + - name: queued.current + type: long + description: > + Current queued requests. For backends, this field reports the number + of requests queued without a server assigned. + + - name: queued.max + type: long + description: > + Maximum value of queued.current. + + - name: errors + type: long + description: > + Request errors. Some of the possible causes are: + + * early termination from the client, before the request has been sent + * read error from the client + * client timeout + * client closed connection + * various bad requests from the client. + * request was tarpitted. + + - name: redispatched + type: long + description: > + Number of times a request was redispatched to another server. For + servers, this field reports the number of times the server was + switched away from. + + - name: connection.errors + type: long + description: > + Number of requests that encountered an error trying to + connect to a server. For backends, this field reports the sum of + the stat for all backend servers, plus any connection errors not + associated with a particular server (such as the backend having no + active servers). + + - name: rate + type: group + description: > + fields: + - name: value + type: long + description: > + Number of HTTP requests per second over the last elapsed second. + - name: max + type: long + description: > + Maximum number of HTTP requests per second. + + - name: total + type: long + description: > + Total number of HTTP requests received. + + + - name: response + type: group + fields: + + - name: errors + type: long + description: > + Number of response errors. This value includes the number of data + transfers aborted by the server (haproxy.stat.server.aborted). + Some other errors are: + + * write errors on the client socket (won't be counted for the server stat) + * failure applying filters to the response + + + - name: time.avg + type: long + description: > + Average response time in ms over the last 1024 requests (0 for TCP). + + - name: denied + type: integer + description: > + Responses denied because of security concerns. For HTTP this is + because of a matched http-request rule, or "option checkcache". + + - name: http + type: group + description: > + + fields: + - name: 1xx + type: long + description: > + HTTP responses with 1xx code. + + - name: 2xx + type: long + description: > + HTTP responses with 2xx code. + + - name: 3xx + type: long + description: > + HTTP responses with 3xx code. + + - name: 4xx + type: long + description: > + HTTP responses with 4xx code. + + - name: 5xx + type: long + description: > + HTTP responses with 5xx code. + + - name: other + type: long + description: > + HTTP responses with other codes (protocol error). + + + - name: session + type: group + fields: + + - name: current + type: long + description: > + Number of current sessions. + + - name: max + type: long + description: > + Maximum number of sessions. + + - name: limit + type: long + description: > + Configured session limit. + + - name: rate + type: group + fields: + - name: value + type: integer + description: > + Number of sessions per second over the last elapsed second. + + - name: limit + type: integer + description: > + Configured limit on new sessions per second. + + - name: max + type: integer + description: > + Maximum number of new sessions per second. + + + - name: check + type: group + description: > + + fields: + - name: status + type: keyword + description: > + Status of the last health check. One of: + + UNK -> unknown + INI -> initializing + SOCKERR -> socket error + L4OK -> check passed on layer 4, no upper layers testing enabled + L4TOUT -> layer 1-4 timeout + L4CON -> layer 1-4 connection problem, for example + "Connection refused" (tcp rst) or "No route to host" (icmp) + L6OK -> check passed on layer 6 + L6TOUT -> layer 6 (SSL) timeout + L6RSP -> layer 6 invalid response - protocol error + L7OK -> check passed on layer 7 + L7OKC -> check conditionally passed on layer 7, for example 404 with + disable-on-404 + L7TOUT -> layer 7 (HTTP/SMTP) timeout + L7RSP -> layer 7 invalid response - protocol error + L7STS -> layer 7 response error, for example HTTP 5xx + + - name: code + type: long + description: > + Layer 5-7 code, if available. + + - name: duration + type: long + description: > + Time in ms that it took to finish the last health check. + + - name: health.last + type: keyword + description: > + The result of the last health check. + + - name: health.fail + type: long + description: > + Number of failed checks. + + - name: agent.last + type: integer + description: > + + - name: failed + type: long + description: > + Number of checks that failed while the server was up. + + - name: down + type: long + description: > + Number of UP->DOWN transitions. For backends, this value is the + number of transitions to the whole backend being down, rather than + the sum of the transitions for each server. + + - name: client.aborted + type: integer + description: > + Number of data transfers aborted by the client. + + + - name: server + type: group + description: > + fields: + + - name: id + type: integer + description: > + Server ID (unique inside a proxy). + + - name: aborted + type: integer + description: > + Number of data transfers aborted by the server. This value is + included in haproxy.stat.response.errors. + + - name: active + type: integer + description: > + Number of backend servers that are active, meaning that they are + healthy and can receive requests from the load balancer. + + - name: backup + type: integer + description: > + Number of backend servers that are backup servers. + + + - name: compressor + type: group + description: > + + fields: + - name: in.bytes + type: long + format: bytes + description: > + Number of HTTP response bytes fed to the compressor. + + - name: out.bytes + type: integer + format: bytes + description: > + Number of HTTP response bytes emitted by the compressor. + + - name: bypassed.bytes + type: long + format: bytes + description: > + Number of bytes that bypassed the HTTP compressor (CPU/BW limit). + + - name: response.bytes + type: long + format: bytes + description: > + Number of HTTP responses that were compressed. + + - name: proxy + type: group + description: > + + fields: + - name: id + type: integer + description: > + Unique proxy ID. + + - name: name + type: keyword + description: > + Proxy name. + + + - name: queue + type: group + description: > + + fields: + - name: limit + type: integer + description: > + Configured queue limit (maxqueue) for the server, or nothing if the + value of maxqueue is 0 (meaning no limit). + + - name: time.avg + type: integer + description: > + The average queue time in ms over the last 1024 requests. + + + + + +- key: http + title: "HTTP" + description: > + HTTP module + fields: + - name: http + type: group + description: > + fields: + - name: request + type: group + description: > + HTTP request information + fields: + - name: header + type: object + description: > + The HTTP headers sent + - name: method + type: keyword + description: > + The HTTP method used + - name: body + type: keyword + description: > + The HTTP payload sent + - name: response + type: group + description: > + HTTP response information + fields: + - name: header + type: object + description: > + The HTTP headers received + - name: status_code + type: keyword + description: > + The HTTP status code + - name: body + type: keyword + description: > + The HTTP payload received + - name: json + type: group + description: > + json metricset + fields: + + - name: server + type: group + description: > + server + fields: + +- key: jolokia + title: "Jolokia" + description: > + beta[] + + Jolokia module + short_config: false + fields: + - name: jolokia + type: group + description: > + jolokia contains metrics exposed via jolokia agent + fields: + +- key: kafka + title: "Kafka" + description: > + Kafka module + + beta[] + short_config: false + fields: + - name: kafka + type: group + description: > + fields: + - name: consumergroup + type: group + description: > + consumergroup + fields: + - name: broker + type: group + description: > + Broker Consumer Group Information have been read from (Broker handling + the consumer group). + fields: + - name: id + type: long + description: > + Broker id + + - name: address + type: keyword + description: > + Broker address + + - name: id + type: keyword + description: Consumer Group ID + + - name: topic + type: keyword + description: Topic name + + - name: partition + type: long + description: Partition ID + + - name: offset + type: long + description: consumer offset into partition being read + + - name: meta + type: text + description: custom consumer meta data string + + - name: error.code + type: long + description: > + kafka consumer/partition error code. + + - name: client + type: group + description: > + Assigned client reading events from partition + fields: + - name: id + type: keyword + description: Client ID (kafka setting client.id) + + - name: host + type: keyword + description: Client host + + - name: member_id + type: keyword + description: internal consumer group member ID + + - name: partition + type: group + description: > + partition + fields: + - name: offset + type: group + description: > + Available offsets of the given partition. + fields: + - name: newest + type: long + description: > + Newest offset of the partition. + - name: oldest + type: long + description: > + Oldest offset of the partition. + + - name: partition + type: group + description: > + Partition data. + fields: + - name: id + type: long + description: > + Partition id. + + - name: leader + type: long + description: > + Leader id (broker). + - name: isr + type: array + description: > + List of isr ids. + - name: replica + type: long + description: > + Replica id (broker). + + - name: insync_replica + type: boolean + description: > + Indicates if replica is included in the in-sync replicate set (ISR). + + - name: error.code + type: long + description: > + Error code from fetching partition. + + - name: topic.error.code + type: long + description: > + topic error code. + - name: topic.name + type: keyword + description: > + Topic name + + - name: broker.id + type: long + description: > + Broker id + - name: broker.address + type: keyword + description: > + Broker address + + + +- key: kibana + title: "Kibana" + description: > + experimental[] + + Kibana module + short_config: false + fields: + - name: kibana + type: group + description: > + fields: + - name: status + type: group + description: > + Status fields + fields: + - name: name + type: keyword + description: > + Kibana instance name. + - name: uuid + type: keyword + description: > + Kibana instance uuid. + - name: version.number + type: keyword + description: > + Kibana version number. + - name: status.overall.state + type: keyword + description: > + Kibana overall state. + - name: metrics + type: group + description: > + Metrics fields + fields: + - name: concurrent_connections + type: long + description: > + Current concurrent connections. + - name: requests + type: group + description: > + Request statistics. + fields: + - name: disconnects + type: long + description: > + Total number of disconnected connections. + - name: total + type: long + description: > + Total number of connections. + +- key: kubernetes + title: "Kubernetes" + description: > + beta[] + + Kubernetes metrics + short_config: false + fields: + - name: kubernetes + type: group + description: > + Information and statistics of pods managed by kubernetes. + fields: + - name: container + type: group + description: > + kubernetes container metrics + fields: + - name: start_time + type: date + description: > + Start time + - name: cpu + type: group + description: > + CPU usage metrics + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + description: > + Container CPU Core usage nanoseconds + - name: nanocores + type: long + description: > + CPU used nanocores + - name: logs + type: group + description: > + Logs info + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Logs available capacity in bytes + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Logs total capacity in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Logs used capacity in bytes + - name: inodes + type: group + fields: + - name: count + type: long + description: > + Total available inodes + - name: free + type: long + description: > + Total free inodes + - name: used + type: long + description: > + Total used inodes + - name: memory + type: group + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total available memory + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total memory usage + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Working set memory usage + - name: pagefaults + type: long + description: > + Number of page faults + - name: majorpagefaults + type: long + description: > + Number of major page faults + - name: rootfs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Root filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Root filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Root filesystem total used in bytes + - name: inodes + type: group + fields: + - name: used + type: long + description: > + Used inodes + + - name: event + type: group + description: > + The Kubernetes events metricset collects events that are generated by objects running + inside of Kubernetes + fields: + - name: count + type: long + description: > + Count field records the number of times the particular event has occured + fields: + - name: timestamp + type: group + fields: + - name: first_occurrence + type: date + description: > + Timestamp of first occurrence of event + - name: last_occurrence + type: date + description: > + Timestamp of last occurrence of event + - name: message + type: keyword + description: > + Message recorded for the given event + - name: reason + type: keyword + description: > + Reason recorded for the given event + - name: type + type: keyword + description: > + Type of the given event + - name: metadata + type: group + description: > + Metadata associated with the given event + fields: + - name: timestamp + type: group + fields: + - name: created + type: date + description: > + Timestamp of creation of the given event + - name: name + type: keyword + description: > + Name of the event + - name: namespace + type: keyword + description: > + Namespace in which event was generated + - name: resource_version + type: keyword + description: > + Version of the event resource + - name: uid + type: keyword + description: > + Unique identifier to the event object + - name: self_link + type: keyword + description: > + URL representing the event + - name: involved_object + type: group + description: > + Metadata associated with the given involved object + fields: + - name: api_version + type: keyword + description: > + API version of the object + - name: kind + type: keyword + description: > + API kind of the object + - name: name + type: keyword + description: > + name of the object + - name: resource_version + type: keyword + description: > + resource version of the object + - name: uid + type: keyword + description: > + UUID version of the object + + - name: node + type: group + description: > + kubernetes node metrics + fields: + - name: name + type: keyword + description: > + Node name + - name: start_time + type: date + description: > + Start time + - name: cpu + type: group + description: > + CPU usage metrics + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + description: > + Node CPU Core usage nanoseconds + - name: nanocores + type: long + description: > + CPU used nanocores + - name: memory + type: group + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total available memory + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total memory usage + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Working set memory usage + - name: pagefaults + type: long + description: > + Number of page faults + - name: majorpagefaults + type: long + description: > + Number of major page faults + - name: network + type: group + fields: + - name: rx + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Received bytes + - name: errors + type: long + description: > + Rx errors + - name: tx + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Transmitted bytes + - name: errors + type: long + description: > + Tx errors + - name: fs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total used in bytes + - name: inodes + type: group + fields: + - name: used + type: long + description: > + Number of used inodes + - name: count + type: long + description: > + Number of inodes + - name: free + type: long + description: > + Number of free inodes + - name: runtime + type: group + fields: + - name: imagefs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Image filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Image filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Image filesystem total used in bytes + + - name: pod + type: group + description: > + kubernetes pod metrics + fields: + - name: start_time + type: date + description: > + Start time + - name: network + type: group + fields: + - name: rx + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Received bytes + - name: errors + type: long + description: > + Rx errors + - name: tx + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Transmitted bytes + - name: errors + type: long + description: > + Tx errors + + - name: container + type: group + description: > + kubernetes container metrics + fields: + - name: id + type: keyword + description: + Container id + - name: status + type: group + fields: + - name: phase + type: keyword + description: > + Container phase (running, waiting, terminated) + - name: ready + type: boolean + description: > + Container ready status + - name: restarts + type: integer + description: > + Container restarts count + - name: cpu + type: group + fields: + - name: limit.nanocores + type: long + description: > + Container CPU nanocores limit + - name: request.nanocores + type: long + description: > + Container CPU requested nanocores + - name: memory + type: group + fields: + - name: limit.bytes + type: long + format: bytes + description: > + Container memory limit in bytes + - name: request.bytes + type: long + format: bytes + description: > + Container requested memory in bytes + + - name: deployment + type: group + description: > + kubernetes deployment metrics + fields: + - name: name + type: keyword + description: > + Kubernetes deployment name + - name: paused + type: boolean + description: > + Kubernetes deployment paused status + - name: replicas + type: group + description: > + Kubernetes deployment replicas info + fields: + - name: desired + type: integer + description: > + Deployment number of desired replicas (spec) + - name: available + type: integer + description: > + Deployment available replicas + - name: unavailable + type: integer + description: > + Deployment unavailable replicas + - name: updated + type: integer + description: > + Deployment updated replicas + + - name: node + type: group + description: > + kubernetes node metrics + fields: + - name: status + type: group + fields: + - name: ready + type: keyword + description: > + Node ready status (true, false or unknown) + - name: unschedulable + type: boolean + description: > + Node unschedulable status + - name: cpu + type: group + fields: + - name: allocatable.cores + type: long + description: > + Node CPU allocatable cores + - name: capacity.cores + type: long + description: > + Node CPU capacity cores + - name: memory + type: group + fields: + - name: allocatable.bytes + type: long + format: bytes + description: > + Node allocatable memory in bytes + - name: capacity.bytes + type: long + format: bytes + description: > + Node memory capacity in bytes + - name: pod + type: group + fields: + - name: allocatable.total + type: long + description: > + Node allocatable pods + - name: capacity.total + type: long + description: > + Node pod capacity + + - name: pod + type: group + description: > + kubernetes pod metrics + fields: + - name: ip + type: ip + description: > + Kubernetes pod IP + - name: host_ip + type: ip + description: > + Kubernetes pod host IP + - name: status + type: group + description: > + Kubernetes pod status metrics + fields: + - name: phase + type: keyword + description: > + Kubernetes pod phase (Running, Pending...) + - name: ready + type: keyword + description: > + Kubernetes pod ready status (true, false or unknown) + - name: scheduled + type: keyword + description: > + Kubernetes pod scheduled status (true, false, unknown) + + - name: replicaset + type: group + description: > + kubernetes replica set metrics + fields: + - name: name + type: keyword + description: > + Kubernetes replica set name + - name: replicas + type: group + description: > + Kubernetes replica set paused status + fields: + - name: available + type: long + description: > + The number of replicas per ReplicaSet + - name: desired + type: long + description: > + The number of replicas per ReplicaSet + - name: ready + type: long + description: > + The number of ready replicas per ReplicaSet + - name: observed + type: long + description: > + The generation observed by the ReplicaSet controller + - name: labeled + type: long + description: > + The number of fully labeled replicas per ReplicaSet + + - name: system + type: group + description: > + kubernetes system containers metrics + fields: + - name: container + type: keyword + description: > + Container name + - name: start_time + type: date + description: > + Start time + - name: cpu + type: group + description: > + CPU usage metrics + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + description: > + CPU Core usage nanoseconds + - name: nanocores + type: long + description: > + CPU used nanocores + - name: memory + type: group + fields: + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total memory usage + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Working set memory usage + - name: pagefaults + type: long + description: > + Number of page faults + - name: majorpagefaults + type: long + description: > + Number of major page faults + + - name: volume + type: group + description: > + kubernetes volume metrics + fields: + - name: name + type: keyword + description: > + Volume name + - name: fs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total used in bytes + - name: inodes + type: group + fields: + - name: used + type: long + description: > + Used inodes + - name: free + type: long + description: > + Free inodes + - name: count + type: long + description: > + Total inodes + +- key: memcached + title: "Memcached" + description: > + beta[] + + Memcached module + short_config: false + fields: + - name: memcached + type: group + description: > + fields: + - name: stats + type: group + description: > + stats + fields: + - name: pid + type: long + description: > + Current process ID of the Memcached task. + + - name: uptime.sec + type: long + description: > + Memcached server uptime. + + - name: threads + type: long + description: > + Number of threads used by the current Memcached server process. + + - name: connections.current + type: long + description: > + Number of open connections to this Memcached server, should be the same + value on all servers during normal operation. + + - name: connections.total + type: long + description: > + Numer of successful connect attempts to this server since it has been started. + + - name: get.hits + type: long + description: > + Number of successful "get" commands (cache hits) since startup, divide them + by the "cmd_get" value to get the cache hitrate. + + - name: get.misses + type: long + description: > + Number of failed "get" requests because nothing was cached for this key + or the cached value was too old. + + - name: cmd.get + type: long + description: > + Number of "get" commands received since server startup not counting if they + were successful or not. + + - name: cmd.set + type: long + description: > + Number of "set" commands serviced since startup. + + - name: read.bytes + type: long + formate: bytes + description: > + Total number of bytes received from the network by this server. + + - name: written.bytes + type: long + formate: bytes + description: > + Total number of bytes send to the network by this server. + + - name: items.current + type: long + description: > + Number of items currently in this server's cache. + + - name: items.total + type: long + formate: bytes + description: > + Number of items stored ever stored on this server. This is no "maximum item + count" value but a counted increased by every new item stored in the cache. + + - name: evictions + type: long + formate: bytes + description: > + Number of objects removed from the cache to free up memory for new items + because Memcached reached it's maximum memory setting (limit_maxbytes). + +- key: mongodb + title: "MongoDB" + description: > + Metrics collected from MongoDB servers. + short_config: false + fields: + - name: mongodb + type: group + description: > + MongoDB metrics. + fields: + - name: dbstats + type: group + description: > + dbstats provides an overview of a particular mongo database. This document + is most concerned with data volumes of a database. + fields: + - name: avg_obj_size.bytes + type: long + format: bytes + + - name: collections + type: integer + + - name: data_size.bytes + type: long + format: bytes + + - name: db + type: keyword + + - name: file_size.bytes + type: long + format: bytes + + - name: index_size.bytes + type: long + format: bytes + + - name: indexes + type: long + + - name: num_extents + type: long + + - name: objects + type: long + + - name: storage_size.bytes + type: long + format: bytes + + - name: ns_size_mb.mb + type: long + + - name: data_file_version + type: group + fields: + - name: major + type: long + + - name: minor + type: long + + - name: extent_free_list + type: group + fields: + - name: num + type: long + + - name: size.bytes + type: long + format: bytes + + - name: status + type: group + description: > + MongoDB server status metrics. + fields: + - name: version + type: keyword + description: > + Instance version. + - name: uptime.ms + type: long + description: > + Instance uptime in milliseconds. + - name: local_time + type: date + description: > + Local time as reported by the MongoDB instance. + + - name: asserts.regular + type: long + description: > + Number of regular assertions produced by the server. + - name: asserts.warning + type: long + description: > + Number of warning assertions produced by the server. + - name: asserts.msg + type: long + description: > + Number of msg assertions produced by the server. + - name: asserts.user + type: long + description: > + Number of user assertions produced by the server. + - name: asserts.rollovers + type: long + description: > + Number of rollovers assertions produced by the server. + + - name: background_flushing + type: group + description: > + Data about the process MongoDB uses to write data to disk. This data is + only available for instances that use the MMAPv1 storage engine. + fields: + - name: flushes + type: long + description: > + A counter that collects the number of times the database has + flushed all writes to disk. + - name: total.ms + type: long + description: > + The total number of milliseconds (ms) that the mongod processes have + spent writing (i.e. flushing) data to disk. Because this is an + absolute value, consider the value of `flushes` and `average_ms` to + provide better context for this datum. + - name: average.ms + type: long + description: > + The average time spent flushing to disk per flush event. + - name: last.ms + type: long + description: > + The amount of time, in milliseconds, that the last flush operation + took to complete. + - name: last_finished + type: date + description: > + A timestamp of the last completed flush operation. + + - name: connections + type: group + description: > + Data regarding the current status of incoming connections and + availability of the database server. + fields: + - name: current + type: long + description: > + The number of connections to the database server from clients. This + number includes the current shell session. Consider the value of + `available` to add more context to this datum. + - name: available + type: long + description: > + The number of unused available incoming connections the database + can provide. + - name: total_created + type: long + description: > + A count of all incoming connections created to the server. This + number includes connections that have since closed. + + - name: journaling + type: group + description: > + Data about the journaling-related operations and performance. Journaling + information only appears for mongod instances that use the MMAPv1 + storage engine and have journaling enabled. + fields: + - name: commits + type: long + description: > + The number of transactions written to the journal during the last + journal group commit interval. + - name: journaled.mb + type: long + description: > + The amount of data in megabytes (MB) written to journal during the + last journal group commit interval. + - name: write_to_data_files.mb + type: long + description: > + The amount of data in megabytes (MB) written from journal to the + data files during the last journal group commit interval. + - name: compression + type: long + description: > + The compression ratio of the data written to the journal. + - name: commits_in_write_lock + type: long + description: > + Count of the commits that occurred while a write lock was held. + Commits in a write lock indicate a MongoDB node under a heavy write + load and call for further diagnosis. + - name: early_commits + type: long + description: > + The number of times MongoDB requested a commit before the scheduled + journal group commit interval. + - name: times + type: group + description: > + Information about the performance of the mongod instance during the + various phases of journaling in the last journal group commit + interval. + fields: + - name: dt.ms + type: long + description: > + The amount of time over which MongoDB collected the times data. + Use this field to provide context to the other times field values. + - name: prep_log_buffer.ms + type: long + description: > + The amount of time spent preparing to write to the journal. + Smaller values indicate better journal performance. + - name: write_to_journal.ms + type: long + description: > + The amount of time spent actually writing to the journal. File + system speeds and device interfaces can affect performance. + - name: write_to_data_files.ms + type: long + description: > + The amount of time spent writing to data files after journaling. + File system speeds and device interfaces can affect performance. + - name: remap_private_view.ms + type: long + description: > + The amount of time spent remapping copy-on-write memory mapped + views. Smaller values indicate better journal performance. + - name: commits.ms + type: long + description: > + The amount of time spent for commits. + - name: commits_in_write_lock.ms + type: long + description: > + The amount of time spent for commits that occurred while a write + lock was held. + + - name: extra_info + type: group + description: > + Platform specific data. + fields: + - name: heap_usage.bytes + type: long + format: bytes + description: > + The total size in bytes of heap space used by the database process. + Only available on Unix/Linux. + - name: page_faults + type: long + description: > + The total number of page faults that require disk operations. Page + faults refer to operations that require the database server to + access data that isn't available in active memory. + + - name: network + type: group + description: > + Platform specific data. + fields: + - name: in.bytes + type: long + format: bytes + description: > + The amount of network traffic, in bytes, received by this database. + - name: out.bytes + type: long + format: bytes + description: > + The amount of network traffic, in bytes, sent from this database. + - name: requests + type: long + description: > + The total number of requests received by the server. + + - name: opcounters + type: group + description: > + An overview of database operations by type. + fields: + - name: insert + type: long + description: > + The total number of insert operations received since the mongod + instance last started. + - name: query + type: long + description: > + The total number of queries received since the mongod instance last + started. + - name: update + type: long + description: > + The total number of update operations received since the mongod + instance last started. + - name: delete + type: long + description: > + The total number of delete operations received since the mongod + instance last started. + - name: getmore + type: long + description: > + The total number of getmore operations received since the mongod + instance last started. + - name: command + type: long + description: > + The total number of commands issued to the database since the mongod + instance last started. + + - name: opcounters_replicated + type: group + description: > + An overview of database replication operations by type. + fields: + - name: insert + type: long + description: > + The total number of replicated insert operations received since the + mongod instance last started. + - name: query + type: long + description: > + The total number of replicated queries received since the mongod + instance last started. + - name: update + type: long + description: > + The total number of replicated update operations received since the + mongod instance last started. + - name: delete + type: long + description: > + The total number of replicated delete operations received since the + mongod instance last started. + - name: getmore + type: long + description: > + The total number of replicated getmore operations received since the + mongod instance last started. + - name: command + type: long + description: > + The total number of replicated commands issued to the database since + the mongod instance last started. + + - name: memory + type: group + description: > + Data about the current memory usage of the mongod server. + fields: + - name: bits + type: long + description: > + Either 64 or 32, depending on which target architecture was specified + during the mongod compilation process. + - name: resident.mb + type: long + description: > + The amount of RAM, in megabytes (MB), currently used by the database + process. + - name: virtual.mb + type: long + description: > + The amount, in megabytes (MB), of virtual memory used by the mongod + process. + - name: mapped.mb + type: long + description: > + The amount of mapped memory, in megabytes (MB), used by the database. + Because MongoDB uses memory-mapped files, this value is likely to be + to be roughly equivalent to the total size of your database or + databases. + - name: mapped_with_journal.mb + type: long + description: > + The amount of mapped memory, in megabytes (MB), including the memory + used for journaling. + - name: write_backs_queued + type: boolean + description: > + True when there are operations from a mongos instance queued for retrying. + - name: storage_engine.name + type: keyword + description: > + A string that represents the name of the current storage engine. + + - name: wired_tiger + type: group + description: > + Statistics about the WiredTiger storage engine. + fields: + - name: concurrent_transactions + type: group + description: > + Statistics about the transactions currently in progress. + fields: + - name: write.out + type: long + description: > + Number of concurrent write transaction in progress. + - name: write.available + type: long + description: > + Number of concurrent write tickets available. + - name: write.total_tickets + type: long + description: > + Number of total write tickets. + - name: read.out + type: long + description: > + Number of concurrent read transaction in progress. + - name: read.available + type: long + description: > + Number of concurrent read tickets available. + - name: read.total_tickets + type: long + description: > + Number of total read tickets. + - name: cache + type: group + description: > + Statistics about the cache and page evictions from the cache. + fields: + - name: maximum.bytes + type: long + format: bytes + description: > + Maximum cache size. + - name: used.bytes + type: long + format: bytes + description: > + Size in byte of the data currently in cache. + - name: dirty.bytes + type: long + format: bytes + description: > + Size in bytes of the dirty data in the cache. + - name: pages.read + type: long + description: > + Number of pages read into the cache. + - name: pages.write + type: long + description: > + Number of pages written from the cache. + - name: pages.evicted + type: long + description: > + Number of pages evicted from the cache. + - name: log + type: group + description: > + Statistics about the write ahead log used by WiredTiger. + fields: + - name: size.bytes + type: long + format: bytes + description: > + Total log size in bytes. + - name: write.bytes + type: long + format: bytes + description: > + Number of bytes written into the log. + - name: max_file_size.bytes + type: long + format: bytes + description: > + Maximum file size. + - name: flushes + type: long + description: > + Number of flush operations. + - name: writes + type: long + description: > + Number of write operations. + - name: scans + type: long + description: > + Number of scan operations. + - name: syncs + type: long + description: > + Number of sync operations. + +- key: mysql + title: "MySQL" + description: > + MySQL server status metrics collected from MySQL. + short_config: false + fields: + - name: mysql + type: group + description: > + `mysql` contains the metrics that were obtained from MySQL + query. + fields: + - name: status + type: group + description: > + `status` contains the metrics that were obtained by the status SQL query. + fields: + - name: aborted + type: group + description: > + Aborted status fields. + fields: + - name: clients + type: long + description: > + The number of connections that were aborted because the client died without closing the connection properly. + + - name: connects + type: long + description: > + The number of failed attempts to connect to the MySQL server. + + - name: binlog + type: group + description: > + fields: + - name: cache.disk_use + type: long + description: > + + - name: cache.use + type: long + description: > + + - name: bytes + type: group + description: > + Bytes stats. + fields: + - name: received + format: bytes + type: long + description: > + The number of bytes received from all clients. + + - name: sent + type: long + format: bytes + description: > + The number of bytes sent to all clients. + + - name: threads + type: group + description: > + Threads stats. + fields: + - name: cached + type: long + description: > + The number of cached threads. + + - name: created + type: long + description: > + The number of created threads. + + - name: connected + type: long + description: > + The number of connected threads. + + - name: running + type: long + description: > + The number of running threads. + + - name: connections + type: long + description: > + + - name: created + type: group + description: > + fields: + - name: tmp.disk_tables + type: long + description: > + + - name: tmp.files + type: long + description: > + + - name: tmp.tables + type: long + description: > + + - name: delayed + type: group + description: > + fields: + - name: errors + type: long + description: > + + - name: insert_threads + type: long + description: > + + - name: writes + type: long + description: > + + - name: flush_commands + type: long + description: > + + - name: max_used_connections + type: long + description: > + + - name: open + type: group + description: > + fields: + - name: files + type: long + description: > + + - name: streams + type: long + description: > + + - name: tables + type: long + description: > + + - name: opened_tables + type: long + description: > + + - name: command + type: group + description: > + fields: + - name: delete + type: long + description: > + The number of DELETE queries since startup. + + - name: insert + type: long + description: > + The number of INSERT queries since startup. + + - name: select + type: long + description: > + The number of SELECT queries since startup. + + - name: update + type: long + description: > + The number of UPDATE queries since startup. + +- key: nginx + title: "Nginx" + description: > + Nginx server status metrics collected from various modules. + short_config: false + fields: + - name: nginx + type: group + description: > + `nginx` contains the metrics that were scraped from nginx. + fields: + - name: stubstatus + type: group + description: > + `stubstatus` contains the metrics that were scraped from the ngx_http_stub_status_module status page. + fields: + - name: hostname + type: keyword + description: > + Nginx hostname. + - name: active + type: long + description: > + The current number of active client connections including Waiting connections. + - name: accepts + type: long + description: > + The total number of accepted client connections. + - name: handled + type: long + description: > + The total number of handled client connections. + - name: dropped + type: long + description: > + The total number of dropped client connections. + - name: requests + type: long + description: > + The total number of client requests. + - name: current + type: long + description: > + The current number of client requests. + - name: reading + type: long + description: > + The current number of connections where Nginx is reading the request header. + - name: writing + type: long + description: > + The current number of connections where Nginx is writing the response back to the client. + - name: waiting + type: long + description: > + The current number of idle client connections waiting for a request. + +- key: php_fpm + title: "PHP_FPM" + description: > + beta[] + + PHP-FPM server status metrics collected from PHP-FPM. + short_config: false + fields: + - name: php_fpm + type: group + description: > + `php_fpm` contains the metrics that were obtained from PHP-FPM status + page call. + fields: + - name: pool + type: group + description: > + `pool` contains the metrics that were obtained from the PHP-FPM process + pool. + fields: + - name: name + type: keyword + description: > + The name of the pool. + - name: connections + type: group + description: > + Connection state specific statistics. + fields: + - name: accepted + type: long + description: > + The number of incoming requests that the PHP-FPM server has accepted; + when a connection is accepted it is removed from the listen queue. + - name: queued + type: long + description: > + The current number of connections that have been initiated, but not + yet accepted. If this value is non-zero it typically means that all + the available server processes are currently busy, and there are no + processes available to serve the next request. Raising + `pm.max_children` (provided the server can handle it) should help + keep this number low. This property follows from the fact that + PHP-FPM listens via a socket (TCP or file based), and thus inherits + some of the characteristics of sockets. + - name: processes + type: group + description: > + Process state specific statistics. + fields: + - name: idle + type: long + description: > + The number of servers in the `waiting to process` state (i.e. not + currently serving a page). This value should fall between the + `pm.min_spare_servers` and `pm.max_spare_servers` values when the + process manager is `dynamic`. + - name: active + type: long + description: > + The number of servers current processing a page - the minimum is `1` + (so even on a fully idle server, the result will be not read `0`). + - name: slow_requests + type: long + description: > + The number of times a request execution time has exceeded + `request_slowlog_timeout`. + +- key: postgresql + title: "PostgreSQL" + description: > + Metrics collected from PostgreSQL servers. + short_config: false + fields: + - name: postgresql + type: group + description: > + PostgreSQL metrics. + fields: + - name: activity + type: group + description: > + One document per server process, showing information related to the current + activity of that process, such as state and current query. Collected by + querying pg_stat_activity. + fields: + - name: database.oid + type: long + description: > + OID of the database this backend is connected to. + - name: database.name + type: keyword + description: > + Name of the database this backend is connected to. + - name: pid + type: long + description: > + Process ID of this backend. + - name: user.id + type: long + description: > + OID of the user logged into this backend. + - name: user.name + description: > + Name of the user logged into this backend. + - name: application_name + description: > + Name of the application that is connected to this backend. + - name: client.address + description: > + IP address of the client connected to this backend. + - name: client.hostname + description: > + Host name of the connected client, as reported by a reverse DNS lookup of client_addr. + - name: client.port + type: long + description: > + TCP port number that the client is using for communication with this + backend, or -1 if a Unix socket is used. + - name: backend_start + type: date + description: > + Time when this process was started, i.e., when the client connected to + the server. + - name: transaction_start + type: date + description: > + Time when this process' current transaction was started. + - name: query_start + type: date + description: > + Time when the currently active query was started, or if state is not + active, when the last query was started. + - name: state_change + type: date + description: > + Time when the state was last changed. + - name: waiting + type: boolean + description: > + True if this backend is currently waiting on a lock. + - name: state + description: > + Current overall state of this backend. Possible values are: + + * active: The backend is executing a query. + * idle: The backend is waiting for a new client command. + * idle in transaction: The backend is in a transaction, but is not + currently executing a query. + * idle in transaction (aborted): This state is similar to idle in + transaction, except one of the statements in the transaction caused + an error. + * fastpath function call: The backend is executing a fast-path function. + * disabled: This state is reported if track_activities is disabled in this backend. + - name: query + description: > + Text of this backend's most recent query. If state is active this field + shows the currently executing query. In all other states, it shows the + last query that was executed. + + + - name: bgwriter + type: group + description: > + Statistics about the background writer process's activity. Collected using the + pg_stat_bgwriter query. + fields: + - name: checkpoints.scheduled + type: long + description: > + Number of scheduled checkpoints that have been performed. + - name: checkpoints.requested + type: long + description: > + Number of requested checkpoints that have been performed. + - name: checkpoints.times.write.ms + type: float + description: > + Total amount of time that has been spent in the portion of checkpoint + processing where files are written to disk, in milliseconds. + - name: checkpoints.times.sync.ms + type: float + description: > + Total amount of time that has been spent in the portion of checkpoint + processing where files are synchronized to disk, in milliseconds. + - name: buffers.checkpoints + type: long + description: > + Number of buffers written during checkpoints. + - name: buffers.clean + type: long + description: > + Number of buffers written by the background writer. + - name: buffers.clean_full + type: long + description: > + Number of times the background writer stopped a cleaning scan because it + had written too many buffers. + - name: buffers.backend + type: long + description: > + Number of buffers written directly by a backend. + - name: buffers.backend_fsync + type: long + description: > + Number of times a backend had to execute its own fsync call (normally + the background writer handles those even when the backend does its own + write) + - name: buffers.allocated + type: long + description: > + Number of buffers allocated. + - name: stats_reset + type: date + description: > + Time at which these statistics were last reset. + + - name: database + type: group + description: > + One row per database, showing database-wide statistics. Collected by querying + pg_stat_database + fields: + - name: oid + type: long + description: > + OID of the database this backend is connected to. + - name: name + type: keyword + description: > + Name of the database this backend is connected to. + - name: number_of_backends + type: long + description: > + Number of backends currently connected to this database. + - name: transactions.commit + type: long + description: > + Number of transactions in this database that have been committed. + - name: transactions.rollback + type: long + description: > + Number of transactions in this database that have been rolled back. + - name: blocks.read + type: long + description: > + Number of disk blocks read in this database. + - name: blocks.hit + type: long + description: > + Number of times disk blocks were found already in the buffer cache, so + that a read was not necessary (this only includes hits in the PostgreSQL + buffer cache, not the operating system's file system cache). + - name: blocks.time.read.ms + type: long + description: > + Time spent reading data file blocks by backends in this database, in + milliseconds. + - name: blocks.time.write.ms + type: long + description: > + Time spent writing data file blocks by backends in this database, in + milliseconds. + - name: rows.returned + type: long + description: > + Number of rows returned by queries in this database. + - name: rows.fetched + type: long + description: > + Number of rows fetched by queries in this database. + - name: rows.inserted + type: long + description: > + Number of rows inserted by queries in this database. + - name: rows.updated + type: long + description: > + Number of rows updated by queries in this database. + - name: rows.deleted + type: long + description: > + Number of rows deleted by queries in this database. + - name: conflicts + type: long + description: > + Number of queries canceled due to conflicts with recovery in this + database. + - name: temporary.files + type: long + description: > + Number of temporary files created by queries in this database. All + temporary files are counted, regardless of why the temporary file was + created (e.g., sorting or hashing), and regardless of the log_temp_files + setting. + - name: temporary.bytes + type: long + description: > + Total amount of data written to temporary files by queries in this + database. All temporary files are counted, regardless of why the + temporary file was created, and regardless of the log_temp_files + setting. + - name: deadlocks + type: long + description: > + Number of deadlocks detected in this database. + - name: stats_reset + type: date + description: > + Time at which these statistics were last reset. + + +- key: prometheus + title: "Prometheus" + description: > + beta[] + + Stats collected from Prometheus. + short_config: false + fields: + - name: prometheus + type: group + description: > + fields: + + - name: stats + type: group + description: > + Stats about the Prometheus server. + fields: + - name: notifications + type: group + description: > + Notification stats. + fields: + - name: queue_length + type: long + description: > + Current queue length. + - name: dropped + type: long + description: > + Number of dropped queue events. + - name: processes.open_fds + type: long + description: > + Number of open file descriptors. + - name: storage.chunks_to_persist + type: long + description: > + Number of memory chunks that are not yet persisted to disk. + +- key: rabbitmq + title: "RabbitMQ" + description: > + experimental[] + + RabbitMQ module + fields: + - name: rabbitmq + type: group + description: > + fields: + - name: node + type: group + description: > + node + fields: + - name: disk.free.bytes + type: long + description: > + Disk free space in bytes. + format: bytes + - name: disk.free.limit.bytes + type: long + description: > + Point at which the disk alarm will go off. + format: bytes + - name: fd.total + type: long + description: > + File descriptors available. + - name: fd.used + type: long + description: > + Used file descriptors. + - name: gc.num.count + type: long + description: > + Number of GC operations. + - name: gc.reclaimed.bytes + type: long + description: > + GC bytes reclaimed. + format: bytes + - name: io.file_handle.open_attempt.avg.ms + type: long + description: > + File handle open avg time + - name: io.file_handle.open_attempt.count + type: long + description: > + File handle open attempts + - name: io.read.avg.ms + type: long + description: > + File handle read avg time + - name: io.read.bytes + type: long + description: > + Data read in bytes + format: bytes + - name: io.read.count + type: long + description: > + Data read operations + - name: io.reopen.count + type: long + description: > + Data reopen operations + - name: io.seek.avg.ms + type: long + description: > + Data seek avg time + - name: io.seek.count + type: long + description: > + Data seek operations + - name: io.sync.avg.ms + type: long + description: > + Data sync avg time + - name: io.sync.count + type: long + description: > + Data sync operations + - name: io.write.avg.ms + type: long + description: > + Data write avg time + - name: io.write.bytes + type: long + description: > + Data write in bytes + format: bytes + - name: io.write.count + type: long + description: > + Data write operations + - name: mem.limit.bytes + type: long + description: > + Point at which the memory alarm will go off. + format: bytes + - name: mem.used.bytes + type: long + description: > + Memory used in bytes. + - name: mnesia.disk.tx.count + type: long + description: > + Number of Mnesia transactions which have been performed that required writes to disk. + - name: mnesia.ram.tx.count + type: long + description: > + Number of Mnesia transactions which have been performed that did not require writes to disk. + - name: msg.store_read.count + type: long + description: > + Number of messages which have been read from the message store. + - name: msg.store_write.count + type: long + description: > + Number of messages which have been written to the message store. + - name: name + type: keyword + description: > + Node name + - name: proc.total + type: long + description: > + Maximum number of Erlang processes. + - name: proc.used + type: long + description: > + Number of Erlang processes in use. + - name: processors + type: long + description: > + Number of cores detected and usable by Erlang. + - name: queue.index.journal_write.count + type: long + description: > + Number of records written to the queue index journal. + - name: queue.index.read.count + type: long + description: > + Number of records read from the queue index. + - name: queue.index.write.count + type: long + description: > + Number of records written to the queue index. + - name: run.queue + type: long + description: > + Average number of Erlang processes waiting to run. + - name: socket.total + type: long + description: > + File descriptors available for use as sockets. + - name: socket.used + type: long + description: > + File descriptors used as sockets. + - name: type + type: keyword + description: > + Node type. + - name: uptime + type: long + description: > + Node uptime. + + - name: queue + type: group + description: > + queue + fields: + - name: name + type: keyword + description: > + The name of the queue with non-ASCII characters escaped as in C. + - name: vhost + type: keyword + description: > + Virtual host name with non-ASCII characters escaped as in C. + - name: durable + type: boolean + description: > + Whether or not the queue survives server restarts. + - name: auto_delete + type: boolean + description: > + Whether the queue will be deleted automatically when no longer used. + - name: exclusive + type: boolean + description: > + Whether the queue is exclusive (i.e. has owner_pid). + - name: node + type: keyword + description: > + Node name. + - name: state + type: keyword + description: > + The state of the queue. Normally 'running', but may be "{syncing, MsgCount}" if the queue is synchronising. Queues which are located on cluster nodes that are currently down will be shown with a status of 'down'. + - name: arguments.max_priority + type: long + description: > + Maximum number of priority levels for the queue to support. + - name: consumers.count + type: long + description: > + Number of consumers. + - name: consumers.utilisation.pct + type: long + format: percentage + description: > + Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count. + - name: messages.total.count + type: long + description: > + Sum of ready and unacknowledged messages (queue depth). + - name: messages.ready.count + type: long + description: > + Number of messages ready to be delivered to clients. + - name: messages.unacknowledged.count + type: long + description: > + Number of messages delivered to clients but not yet acknowledged. + - name: messages.persistent.count + type: long + description: > + Total number of persistent messages in the queue (will always be 0 for transient queues). + - name: memory.bytes + type: long + format: bytes + description: > + Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures. + - name: disk.reads.count + type: long + description: > + Total number of times messages have been read from disk by this queue since it started. + - name: disk.writes.count + type: long + description: > + Total number of times messages have been written to disk by this queue since it started. + +- key: redis + title: "Redis" + description: > + Redis metrics collected from Redis. + fields: + - name: redis + type: group + description: > + `redis` contains the information and statistics from Redis. + fields: + - name: info + type: group + description: > + `info` contains the information and statistics returned by the `INFO` command. + fields: + - name: clients + type: group + description: > + Redis client stats. + fields: + - name: connected + type: long + description: > + Number of client connections (excluding connections from slaves). + + - name: longest_output_list + type: long + description: > + Longest output list among current client connections. + + - name: biggest_input_buf + type: long + description: > + Biggest input buffer among current client connections. + + - name: blocked + type: long + description: > + Number of clients pending on a blocking call (BLPOP, BRPOP, BRPOPLPUSH). + + - name: cluster + type: group + description: > + Redis cluster information. + fields: + - name: enabled + type: boolean + description: > + Indicates that the Redis cluster is enabled. + + - name: cpu + type: group + description: > + Redis CPU stats + fields: + - name: used.sys + type: scaled_float + description: > + System CPU consumed by the Redis server. + + - name: used.sys_children + type: scaled_float + description: > + User CPU consumed by the Redis server. + + - name: used.user + type: scaled_float + description: > + System CPU consumed by the background processes. + + - name: used.user_children + type: scaled_float + description: > + User CPU consumed by the background processes. + + - name: memory + type: group + description: > + Redis memory stats. + fields: + - name: used.value + type: long + description: > + format: bytes + Used memory. + + - name: used.rss + type: long + format: bytes + description: > + Used memory rss. + + - name: used.peak + type: long + format: bytes + description: > + Used memory peak. + + - name: used.lua + type: long + format: bytes + description: > + Used memory lua. + + - name: allocator + type: keyword + description: > + Memory allocator. + + - name: persistence + type: group + description: > + Redis CPU stats. + fields: + - name: loading + type: boolean + description: + + - name: rdb + type: group + description: + fields: + - name: last_save.changes_since + type: long + description: + + - name: bgsave.in_progress + type: boolean + description: + + - name: last_save.time + type: long + description: + + - name: bgsave.last_status + type: keyword + description: + + - name: bgsave.last_time.sec + type: long + description: + + - name: bgsave.current_time.sec + type: long + description: + + - name: aof + type: group + description: + fields: + - name: enabled + type: boolean + description: + + - name: rewrite.in_progress + type: boolean + description: + + - name: rewrite.scheduled + type: boolean + description: + + - name: rewrite.last_time.sec + type: long + description: + + - name: rewrite.current_time.sec + type: long + description: + + - name: bgrewrite.last_status + type: keyword + description: + + - name: write.last_status + type: keyword + description: + + - name: replication + type: group + description: > + Replication + fields: + - name: role + type: keyword + description: + + - name: connected_slaves + type: long + description: + + - name: master_offset + type: long + description: + + - name: backlog.active + type: long + description: + + - name: backlog.size + type: long + description: + + - name: backlog.first_byte_offset + type: long + description: + + - name: backlog.histlen + type: long + description: + + - name: server + type: group + description: > + Server info + fields: + - name: version + type: keyword + description: + + - name: git_sha1 + type: keyword + description: + + - name: git_dirty + type: keyword + description: + + - name: build_id + type: keyword + description: + + - name: mode + type: keyword + description: + + - name: os + type: keyword + description: + + - name: arch_bits + type: keyword + description: + + - name: multiplexing_api + type: keyword + description: + + - name: gcc_version + type: keyword + description: + + - name: process_id + type: long + description: + + - name: run_id + type: keyword + description: + + - name: tcp_port + type: long + description: + + - name: uptime + type: long + description: + + - name: hz + type: long + description: + + - name: lru_clock + type: long + description: + + - name: config_file + type: keyword + description: + + - name: stats + type: group + description: > + Redis stats. + fields: + - name: connections.received + type: long + description: + Total number of connections received. + + - name: connections.rejected + type: long + description: + Total number of connections rejected. + + - name: commands_processed + type: long + description: + Total number of commands preocessed. + + - name: net.input.bytes + type: long + description: + Total network input in bytes. + + - name: net.output.bytes + type: long + description: + Total network output in bytes. + + - name: instantaneous.ops_per_sec + type: long + description: + + - name: instantaneous.input_kbps + type: scaled_float + description: + + - name: instantaneous.output_kbps + type: scaled_float + description: + + - name: sync.full + type: long + description: + + - name: sync.partial.ok + type: long + description: + + - name: sync.partial.err + type: long + description: + + - name: keys.expired + type: long + description: + + - name: keys.evicted + type: long + description: + + - name: keyspace.hits + type: long + description: + + - name: keyspace.misses + type: long + description: + + - name: pubsub.channels + type: long + description: + + - name: pubsub.patterns + type: long + description: + + - name: latest_fork_usec + type: long + description: + + - name: migrate_cached_sockets + type: long + description: + + + - name: keyspace + type: group + description: > + `keyspace` contains the information about the keyspaces returned by the `INFO` command. + fields: + - name: id + type: keyword + description: > + Keyspace identifier. + + - name: avg_ttl + type: long + description: > + Average ttl. + + - name: keys + type: long + description: > + Number of keys in the keyspace. + + - name: expires + type: long + description: > + +- key: system + title: "System" + description: > + System status metrics, like CPU and memory usage, that are collected from the operating system. + short_config: true + fields: + - name: system + type: group + description: > + `system` contains local system metrics. + fields: + - name: core + type: group + description: > + `system-core` contains CPU metrics for a single core of a multi-core system. + fields: + - name: id + type: long + description: > + CPU Core number. + + # Percentages + - name: user.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. + + - name: user.ticks + type: long + description: > + The amount of CPU time spent in user space. + + - name: system.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: system.ticks + type: long + description: > + The amount of CPU time spent in kernel space. + + - name: nice.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: nice.ticks + type: long + description: > + The amount of CPU time spent on low-priority processes. + + - name: idle.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: idle.ticks + type: long + description: > + The amount of CPU time spent idle. + + - name: iowait.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: iowait.ticks + type: long + description: > + The amount of CPU time spent in wait (on disk). + + - name: irq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: irq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: softirq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling software interrupts. + + - name: steal.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: steal.ticks + type: long + description: > + The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: cpu + type: group + description: > + `cpu` contains local CPU stats. + fields: + - name: cores + type: long + description: > + The number of CPU cores present on the host. The non-normalized + percentages will have a maximum value of `100% * cores`. The + normalized percentages already take this value into account and have + a maximum value of 100%. + + # Percentages + - name: user.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. On multi-core systems, + you can have percentages that are greater than 100%. For example, if 3 + cores are at 60% use, then the `system.cpu.user.pct` will be 180%. + + - name: system.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: nice.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: idle.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: iowait.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: irq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: steal.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: total.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in non-idle state. + + # Normalized Percentages + - name: user.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. + + - name: system.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: nice.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: idle.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: iowait.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: irq.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: steal.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: total.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in non-idle state. + + + # Ticks + - name: user.ticks + type: long + description: > + The amount of CPU time spent in user space. + + - name: system.ticks + type: long + description: > + The amount of CPU time spent in kernel space. + + - name: nice.ticks + type: long + description: > + The amount of CPU time spent on low-priority processes. + + - name: idle.ticks + type: long + description: > + The amount of CPU time spent idle. + + - name: iowait.ticks + type: long + description: > + The amount of CPU time spent in wait (on disk). + + - name: irq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling software interrupts. + + - name: steal.ticks + type: long + description: > + The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: diskio + type: group + description: > + `disk` contains disk IO metrics collected from the operating system. + fields: + - name: name + type: keyword + example: sda1 + description: > + The disk name. + + - name: serial_number + type: keyword + description: > + The disk's serial number. This may not be provided by all operating + systems. + + - name: read.count + type: long + description: > + The total number of reads completed successfully. + + - name: write.count + type: long + description: > + The total number of writes completed successfully. + + - name: read.bytes + type: long + format: bytes + description: > + The total number of bytes read successfully. On Linux this is + the number of sectors read multiplied by an assumed sector size of 512. + + - name: write.bytes + type: long + format: bytes + description: > + The total number of bytes written successfully. On Linux this is + the number of sectors written multiplied by an assumed sector size of + 512. + + - name: read.time + type: long + description: > + The total number of milliseconds spent by all reads. + + - name: write.time + type: long + description: > + The total number of milliseconds spent by all writes. + + - name: io.time + type: long + description: > + The total number of of milliseconds spent doing I/Os. + + - name: iostat.read.request.merges_per_sec + type: float + description: > + The number of read requests merged per second that were queued to the device. + + - name: iostat.write.request.merges_per_sec + type: float + description: > + The number of write requests merged per second that were queued to the device. + + - name: iostat.read.request.per_sec + type: float + description: > + The number of read requests that were issued to the device per second + + - name: iostat.write.request.per_sec + type: float + description: > + The number of write requests that were issued to the device per second + + - name: iostat.read.per_sec.bytes + type: float + description: > + The number of Bytes read from the device per second. + format: bytes + + - name: iostat.write.per_sec.bytes + type: float + description: > + The number of Bytes write from the device per second. + format: bytes + + - name: iostat.request.avg_size + type: float + description: > + The average size (in sectors) of the requests that were issued to the device. + + - name: iostat.queue.avg_size + type: float + description: > + The average queue length of the requests that were issued to the device. + + - name: iostat.await + type: float + description: > + The average queue length of the requests that were issued to the device. + + - name: iostat.service_time + type: float + description: > + The average service time (in milliseconds) for I/O requests that were issued to the device. + + - name: iostat.busy + type: float + description: > + Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. + + - name: filesystem + type: group + description: > + `filesystem` contains local filesystem stats. + fields: + - name: available + type: long + format: bytes + description: > + The disk space available to an unprivileged user in bytes. + - name: device_name + type: keyword + description: > + The disk name. For example: `/dev/disk1` + - name: type + type: keyword + description: > + The disk type. For example: `ext4` + - name: mount_point + type: keyword + description: > + The mounting point. For example: `/` + - name: files + type: long + description: > + The total number of file nodes in the file system. + - name: free + type: long + format: bytes + description: > + The disk space available in bytes. + - name: free_files + type: long + description: > + The number of free file nodes in the file system. + - name: total + type: long + format: bytes + description: > + The total disk space in bytes. + - name: used.bytes + type: long + format: bytes + description: > + The used disk space in bytes. + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used disk space. + + + + - name: fsstat + type: group + description: > + `system.fsstat` contains filesystem metrics aggregated from all mounted + filesystems, similar with what `df -a` prints out. + fields: + - name: count + type: long + description: Number of file systems found. + - name: total_files + type: long + description: Total number of files. + - name: total_size + format: bytes + type: group + description: Nested file system docs. + fields: + - name: free + type: long + format: bytes + description: > + Total free space. + - name: used + type: long + format: bytes + description: > + Total used space. + - name: total + type: long + format: bytes + description: > + Total space (used plus free). + + - name: load + type: group + description: > + CPU load averages. + fields: + - name: "1" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last minute. + - name: "5" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 5 minutes. + - name: "15" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 15 minutes. + + - name: "norm.1" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last minute divided by the number of cores. + + - name: "norm.5" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last 5 minutes divided by the number of cores. + + - name: "norm.15" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last 15 minutes divided by the number of cores. + + - name: "cores" + type: long + description: > + The number of CPU cores present on the host. + + - name: memory + type: group + description: > + `memory` contains local memory stats. + fields: + - name: total + type: long + format: bytes + description: > + Total memory. + + - name: used.bytes + type: long + format: bytes + description: > + Used memory. + + - name: free + type: long + format: bytes + description: > + The total amount of free memory in bytes. This value does not include memory consumed by system caches and + buffers (see system.memory.actual.free). + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used memory. + + - name: actual + type: group + description: > + Actual memory used and free. + fields: + + - name: used.bytes + type: long + format: bytes + description: > + Actual used memory in bytes. It represents the difference between the total and the available memory. The + available memory depends on the OS. For more details, please check `system.actual.free`. + + - name: free + type: long + format: bytes + description: > + Actual free memory in bytes. It is calculated based on the OS. On Linux it consists of the free memory + plus caches and buffers. On OSX it is a sum of free memory and the inactive memory. On Windows, it is equal + to `system.memory.free`. + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of actual used memory. + + - name: swap + type: group + prefix: "[float]" + description: This group contains statistics related to the swap memory usage on the system. + fields: + - name: total + type: long + format: bytes + description: > + Total swap memory. + + - name: used.bytes + type: long + format: bytes + description: > + Used swap memory. + + - name: free + type: long + format: bytes + description: > + Available swap memory. + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used swap memory. + + - name: network + type: group + description: > + `network` contains network IO metrics for a single network interface. + fields: + - name: name + type: keyword + example: eth0 + description: > + The network interface name. + + - name: out.bytes + type: long + format: bytes + description: > + The number of bytes sent. + + - name: in.bytes + type: long + format: bytes + description: > + The number of bytes received. + + - name: out.packets + type: long + description: > + The number of packets sent. + + - name: in.packets + type: long + description: > + The number or packets received. + + - name: in.errors + type: long + description: > + The number of errors while receiving. + + - name: out.errors + type: long + description: > + The number of errors while sending. + + - name: in.dropped + type: long + description: > + The number of incoming packets that were dropped. + + - name: out.dropped + type: long + description: > + The number of outgoing packets that were dropped. This value is always + 0 on Darwin and BSD because it is not reported by the operating system. + + - name: process + type: group + description: > + `process` contains process metadata, CPU metrics, and memory metrics. + fields: + - name: name + type: keyword + description: > + The process name. + - name: state + type: keyword + description: > + The process state. For example: "running". + - name: pid + type: long + description: > + The process pid. + - name: ppid + type: long + description: > + The process parent pid. + - name: pgid + type: long + description: > + The process group id. + - name: cmdline + type: keyword + description: > + The full command-line used to start the process, including the + arguments separated by space. + - name: username + type: keyword + description: > + The username of the user that created the process. If the username + cannot be determined, the field will contain the user's + numeric identifier (UID). On Windows, this field includes the user's + domain and is formatted as `domain\username`. + - name: cwd + type: keyword + description: > + The current working directory of the process. This field is only + available on Linux. + - name: env + type: object + object_type: keyword + description: > + The environment variables used to start the process. The data is + available on FreeBSD, Linux, and OS X. + - name: cpu + type: group + prefix: "[float]" + description: CPU-specific statistics per process. + fields: + - name: user + type: long + description: > + The amount of CPU time the process spent in user space. + - name: total.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent by the process since the last update. Its value is similar to the + %CPU value of the process displayed by the top command on Unix systems. + - name: total.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent by the process since the last event. + This value is normalized by the number of CPU cores and it ranges + from 0 to 100%. + - name: system + type: long + description: > + The amount of CPU time the process spent in kernel space. + - name: total.ticks + type: long + description: > + The total CPU time spent by the process. + - name: start_time + type: date + description: > + The time when the process was started. + - name: memory + type: group + description: Memory-specific statistics per process. + prefix: "[float]" + fields: + - name: size + type: long + format: bytes + description: > + The total virtual memory the process has. + - name: rss.bytes + type: long + format: bytes + description: > + The Resident Set Size. The amount of memory the process occupied in main memory (RAM). + - name: rss.pct + type: scaled_float + format: percent + description: > + The percentage of memory the process occupied in main memory (RAM). + - name: share + type: long + format: bytes + description: > + The shared memory the process uses. + - name: fd + type: group + description: > + File descriptor usage metrics. This set of metrics is available for + Linux and FreeBSD. + prefix: "[float]" + fields: + - name: open + type: long + description: The number of file descriptors open by the process. + - name: limit.soft + type: long + description: > + The soft limit on the number of file descriptors opened by the + process. The soft limit can be changed by the process at any time. + - name: limit.hard + type: long + description: > + The hard limit on the number of file descriptors opened by the + process. The hard limit can only be raised by root. + - name: cgroup + type: group + description: > + Metrics and limits from the cgroup of which the task is a member. + cgroup metrics are reported when the process has membership in a + non-root cgroup. These metrics are only available on Linux. + fields: + - name: id + type: keyword + description: > + The ID common to all cgroups associated with this task. + If there isn't a common ID used by all cgroups this field will be + absent. + + - name: path + type: keyword + description: > + The path to the cgroup relative to the cgroup subsystem's mountpoint. + If there isn't a common path used by all cgroups this field will be + absent. + + - name: cpu + type: group + description: > + The cpu subsystem schedules CPU access for tasks in the cgroup. + Access can be controlled by two separate schedulers, CFS and RT. + CFS stands for completely fair scheduler which proportionally + divides the CPU time between cgroups based on weight. RT stands for + real time scheduler which sets a maximum amount of CPU time that + processes in the cgroup can consume during a given period. + + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's + mountpoint. + + - name: cfs.period.us + type: long + description: > + Period of time in microseconds for how regularly a + cgroup's access to CPU resources should be reallocated. + + - name: cfs.quota.us + type: long + description: > + Total amount of time in microseconds for which all + tasks in a cgroup can run during one period (as defined by + cfs.period.us). + + - name: cfs.shares + type: long + description: > + An integer value that specifies a relative share of CPU time + available to the tasks in a cgroup. The value specified in the + cpu.shares file must be 2 or higher. + + - name: rt.period.us + type: long + description: > + Period of time in microseconds for how regularly a cgroup's + access to CPU resources is reallocated. + + - name: rt.runtime.us + type: long + description: > + Period of time in microseconds for the longest continuous period + in which the tasks in a cgroup have access to CPU resources. + + - name: stats.periods + type: long + description: > + Number of period intervals (as specified in cpu.cfs.period.us) + that have elapsed. + + - name: stats.throttled.periods + type: long + description: > + Number of times tasks in a cgroup have been throttled (that is, + not allowed to run because they have exhausted all of the + available time as specified by their quota). + + - name: stats.throttled.ns + type: long + description: > + The total time duration (in nanoseconds) for which tasks in a + cgroup have been throttled. + + - name: cpuacct + type: group + description: CPU accounting metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's + mountpoint. + + - name: total.ns + type: long + description: > + Total CPU time in nanoseconds consumed by all tasks in the + cgroup. + + - name: stats.user.ns + type: long + description: CPU time consumed by tasks in user mode. + + - name: stats.system.ns + type: long + description: CPU time consumed by tasks in user (kernel) mode. + + - name: percpu + type: object + object_type: long + description: > + CPU time (in nanoseconds) consumed on each CPU by all tasks in + this cgroup. + + - name: memory + type: group + description: Memory limits and metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's mountpoint. + + - name: mem.usage.bytes + type: long + format: bytes + description: > + Total memory usage by processes in the cgroup (in bytes). + + - name: mem.usage.max.bytes + type: long + format: bytes + description: > + The maximum memory used by processes in the cgroup (in bytes). + + - name: mem.limit.bytes + type: long + format: bytes + description: > + The maximum amount of user memory in bytes (including file + cache) that tasks in the cgroup are allowed to use. + + - name: mem.failures + type: long + description: > + The number of times that the memory limit (mem.limit.bytes) was + reached. + + - name: memsw.usage.bytes + type: long + format: bytes + description: > + The sum of current memory usage plus swap space used by + processes in the cgroup (in bytes). + + - name: memsw.usage.max.bytes + type: long + format: bytes + description: > + The maximum amount of memory and swap space used by processes in + the cgroup (in bytes). + + - name: memsw.limit.bytes + type: long + format: bytes + description: > + The maximum amount for the sum of memory and swap usage + that tasks in the cgroup are allowed to use. + + - name: memsw.failures + type: long + description: > + The number of times that the memory plus swap space limit + (memsw.limit.bytes) was reached. + + - name: kmem.usage.bytes + type: long + format: bytes + description: > + Total kernel memory usage by processes in the cgroup (in bytes). + + - name: kmem.usage.max.bytes + type: long + format: bytes + description: > + The maximum kernel memory used by processes in the cgroup (in + bytes). + + - name: kmem.limit.bytes + type: long + format: bytes + description: > + The maximum amount of kernel memory that tasks in the cgroup are + allowed to use. + + - name: kmem.failures + type: long + description: > + The number of times that the memory limit (kmem.limit.bytes) was + reached. + + - name: kmem_tcp.usage.bytes + type: long + format: bytes + description: > + Total memory usage for TCP buffers in bytes. + + - name: kmem_tcp.usage.max.bytes + type: long + format: bytes + description: > + The maximum memory used for TCP buffers by processes in the + cgroup (in bytes). + + - name: kmem_tcp.limit.bytes + type: long + format: bytes + description: > + The maximum amount of memory for TCP buffers that tasks in the + cgroup are allowed to use. + + - name: kmem_tcp.failures + type: long + description: > + The number of times that the memory limit (kmem_tcp.limit.bytes) + was reached. + + - name: stats.active_anon.bytes + type: long + format: bytes + description: > + Anonymous and swap cache on active least-recently-used (LRU) + list, including tmpfs (shmem), in bytes. + + - name: stats.active_file.bytes + type: long + format: bytes + description: File-backed memory on active LRU list, in bytes. + + - name: stats.cache.bytes + type: long + format: bytes + description: Page cache, including tmpfs (shmem), in bytes. + + - name: stats.hierarchical_memory_limit.bytes + type: long + format: bytes + description: > + Memory limit for the hierarchy that contains the memory cgroup, + in bytes. + + - name: stats.hierarchical_memsw_limit.bytes + type: long + format: bytes + description: > + Memory plus swap limit for the hierarchy that contains the + memory cgroup, in bytes. + + - name: stats.inactive_anon.bytes + type: long + format: bytes + description: > + Anonymous and swap cache on inactive LRU list, including tmpfs + (shmem), in bytes + + - name: stats.inactive_file.bytes + type: long + format: bytes + description: > + File-backed memory on inactive LRU list, in bytes. + + - name: stats.mapped_file.bytes + type: long + format: bytes + description: > + Size of memory-mapped mapped files, including tmpfs (shmem), + in bytes. + + - name: stats.page_faults + type: long + description: > + Number of times that a process in the cgroup triggered a page + fault. + + - name: stats.major_page_faults + type: long + description: > + Number of times that a process in the cgroup triggered a major + fault. "Major" faults happen when the kernel actually has to + read the data from disk. + + - name: stats.pages_in + type: long + description: > + Number of pages paged into memory. This is a counter. + + - name: stats.pages_out + type: long + description: > + Number of pages paged out of memory. This is a counter. + + - name: stats.rss.bytes + type: long + format: bytes + description: > + Anonymous and swap cache (includes transparent hugepages), not + including tmpfs (shmem), in bytes. + + - name: stats.rss_huge.bytes + type: long + format: bytes + description: > + Number of bytes of anonymous transparent hugepages. + + - name: stats.swap.bytes + type: long + format: bytes + description: > + Swap usage, in bytes. + + - name: stats.unevictable.bytes + type: long + format: bytes + description: > + Memory that cannot be reclaimed, in bytes. + + - name: blkio + type: group + description: Block IO metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystems mountpoint. + + - name: total.bytes + type: long + format: bytes + description: > + Total number of bytes transferred to and from all block devices + by processes in the cgroup. + + - name: total.ios + type: long + description: > + Total number of I/O operations performed on all devices + by processes in the cgroup as seen by the throttling policy. + + - name: process.summary + title: Process Summary + type: group + description: > + Summary metrics for the processes running on the host. + fields: + - name: total + type: long + description: > + Total number of processes on this host. + - name: running + type: long + description: > + Number of running processes on this host. + - name: idle + type: long + description: > + Number of idle processes on this host. + - name: sleeping + type: long + description: > + Number of sleeping processes on this host. + - name: stopped + type: long + description: > + Number of stopped processes on this host. + - name: zombie + type: long + description: > + Number of zombie processes on this host. + - name: unknown + type: long + description: > + Number of processes for which the state couldn't be retrieved or is unknown. + + - name: socket + type: group + description: > + TCP sockets that are active. + fields: + - name: direction + type: keyword + example: incoming + description: > + How the socket was initiated. Possible values are incoming, outgoing, + or listening. + + - name: family + type: keyword + example: ipv4 + description: > + Address family. + + - name: local.ip + type: ip + example: 192.0.2.1 or 2001:0DB8:ABED:8536::1 + description: > + Local IP address. This can be an IPv4 or IPv6 address. + + - name: local.port + type: long + example: 22 + description: > + Local port. + + - name: remote.ip + type: ip + example: 192.0.2.1 or 2001:0DB8:ABED:8536::1 + description: > + Remote IP address. This can be an IPv4 or IPv6 address. + + - name: remote.port + type: long + example: 22 + description: > + Remote port. + + - name: remote.host + type: keyword + example: 76-211-117-36.nw.example.com. + description: > + PTR record associated with the remote IP. It is obtained via reverse + IP lookup. + + - name: remote.etld_plus_one + type: keyword + example: example.com. + description: > + The effective top-level domain (eTLD) of the remote host plus one more + label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". + The data for determining the eTLD comes from an embedded copy of the data + from http://publicsuffix.org. + + - name: remote.host_error + type: keyword + description: > + Error describing the cause of the reverse lookup failure. + + - name: process.pid + type: long + description: > + ID of the process that opened the socket. + + - name: process.command + type: keyword + description: > + Name of the command (limited to 20 chars by the OS). + + - name: process.cmdline + type: keyword + description: > + + - name: process.exe + type: keyword + description: > + Absolute path to the executable. + + - name: user.id + type: long + description: > + UID of the user running the process. + + - name: user.name + type: keyword + description: > + Name of the user running the process. + + - name: uptime + type: group + description: > + `uptime` contains the operating system uptime metric. + fields: + - name: duration.ms + type: long + format: duration + input_format: milliseconds + description: > + The OS uptime in milliseconds. + +- key: vsphere + title: "vSphere" + description: > + vSphere module + fields: + - name: vsphere + type: group + description: > + fields: + - name: datastore + type: group + description: > + datastore + fields: + - name: datacenter + type: keyword + description: > + Datacenter name + - name: name + type: keyword + description: > + Datastore name + - name: fstype + type: keyword + description: > + Filesystem type + - name: capacity.total.bytes + type: long + description: > + Total bytes of the datastore + format: bytes + - name: capacity.free.bytes + type: long + description: > + Free bytes of the datastore + format: bytes + - name: capacity.used.bytes + type: long + description: > + Used bytes of the datastore + format: bytes + - name: capacity.used.pct + type: long + description: > + Used percent of the datastore + format: percent + + + - name: host + type: group + description: > + host + fields: + - name: datacenter + type: keyword + description: > + Datacenter name + - name: name + type: keyword + description: > + Host name + - name: cpu.used.mhz + type: long + description: > + Used CPU in Mhz + - name: cpu.total.mhz + type: long + description: > + Total CPU in Mhz + - name: cpu.free.mhz + type: long + description: > + Free CPU in Mhz + - name: memory.used.bytes + type: long + description: > + Used Memory in bytes + format: bytes + - name: memory.total.bytes + type: long + description: > + Total Memory in bytes + format: bytes + - name: memory.free.bytes + type: long + description: > + Free Memory in bytes + format: bytes + + - name: virtualmachine + type: group + description: > + virtualmachine + fields: + - name: datacenter + type: keyword + description: > + Datacenter name + - name: name + type: keyword + description: > + Virtual Machine name + - name: cpu.used.mhz + type: long + description: > + Used CPU in Mhz + - name: memory.used.guest.bytes + type: long + description: > + Used Memory of Guest in bytes + format: bytes + - name: memory.used.host.bytes + type: long + description: > + Used Memory of Host in bytes + format: bytes + - name: memory.total.guest.bytes + type: long + description: > + Total Memory of Guest in bytes + format: bytes + - name: memory.free.guest.bytes + type: long + description: > + Free Memory of Guest in bytes + format: bytes + - name: custom_fields + type: object + object_type: keyword + descripton: > + Custom fields + +- key: windows + title: "Windows" + description: > + beta[] + Module for Windows + short_config: false + fields: + - name: windows + type: group + description: > + fields: + +- key: zookeeper + title: "ZooKeeper" + description: > + ZooKeeper metrics collected by the four-letter monitoring commands. + short_config: false + fields: + - name: zookeeper + type: group + description: > + `zookeeper` contains the metrics reported by ZooKeeper + commands. + fields: + - name: mntr + type: group + description: > + `mntr` contains the metrics reported by the four-letter `mntr` + command. + fields: + - name: hostname + type: keyword + description: > + ZooKeeper hostname. + - name: approximate_data_size + type: long + description: > + Approximate size of ZooKeeper data. + - name: latency.avg + type: long + description: > + Average latency between ensemble hosts in milliseconds. + - name: ephemerals_count + type: long + description: > + Number of ephemeral znodes. + - name: followers + type: long + description: > + Number of followers seen by the current host. + - name: max_file_descriptor_count + type: long + description: > + Maximum number of file descriptors allowed for the ZooKeeper process. + - name: latency.max + type: long + description: > + Maximum latency in milliseconds. + - name: latency.min + type: long + description: > + Minimum latency in milliseconds. + - name: num_alive_connections + type: long + description: > + Number of connections to ZooKeeper that are currently alive. + - name: open_file_descriptor_count + type: long + description: > + Number of file descriptors open by the ZooKeeper process. + - name: outstanding_requests + type: long + description: > + Number of outstanding requests that need to be processed by the cluster. + - name: packets.received + type: long + description: > + Number of ZooKeeper network packets received. + - name: packets.sent + type: long + description: > + Number of ZooKeeper network packets sent. + - name: pending_syncs + type: long + description: > + Number of pending syncs to carry out to ZooKeeper ensemble followers. + - name: server_state + type: keyword + description: > + Role in the ZooKeeper ensemble. + - name: synced_followers + type: long + description: > + Number of synced followers reported when a node server_state is leader. + - name: version + type: keyword + description: > + ZooKeeper version and build string reported. + - name: watch_count + type: long + description: > + Number of watches currently set on the local ZooKeeper process. + - name: znode_count + type: long + description: > + Number of znodes reported by the local ZooKeeper process. + + diff --git a/libbeat/kibana/testdata/metricbeat-5x-old.json b/libbeat/kibana/testdata/metricbeat-5x-old.json new file mode 100644 index 00000000000..e81c0b2d00a --- /dev/null +++ b/libbeat/kibana/testdata/metricbeat-5x-old.json @@ -0,0 +1,6 @@ +{ + "fields": "[{\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.timezone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"@timestamp\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"fields\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"error.message\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"error.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.provider\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.machine_type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.availability_zone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.project_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.region\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.image\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.namespace\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.annotations\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.image\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.module\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.host\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.rtt\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.namespace\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.delete.error\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.delete.not_found\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.delete.success\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.delete.timeout\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.read.error\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.read.not_found\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.read.success\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.read.timeout\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.write.error\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.write.success\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.write.timeout\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.device.available.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.device.free.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.device.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.device.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.hwm_breached\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.free.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.used.data.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.used.index.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.used.sindex.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.used.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.node.host\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.node.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.objects.master\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.objects.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.stop_writes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.total_accesses\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.total_kbytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.requests_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.bytes_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.bytes_per_request\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.workers.busy\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.workers.idle\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.uptime.server_uptime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.uptime.uptime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.load\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.system\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.children_user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.children_system\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.connections.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.connections.async.writing\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.connections.async.keep_alive\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.connections.async.closing\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.load.1\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.load.5\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.load.15\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.starting_up\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.reading_request\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.sending_reply\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.keepalive\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.dns_lookup\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.closing_connection\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.logging\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.gracefully_finishing\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.idle_cleanup\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.open_slot\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.waiting_for_connection\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_disk.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_disk.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_disk.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_health.overall_status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_health.timechecks.epoch\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_health.timechecks.round.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_health.timechecks.round.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.traffic.read_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.traffic.write_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.traffic.read_op_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.traffic.write_op_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.misplace.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.misplace.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.misplace.ratio\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.degraded.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.degraded.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.degraded.ratio\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg.data_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg.avail_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg.total_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg.used_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg_state.state_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg_state.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg_state.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.full\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.nearfull\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.num_osds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.num_up_osds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.num_in_osds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.num_in_osds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.epoch\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.available.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.health\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.available.kb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.total.kb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.used.kb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.last_updated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.log.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.misc.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.sst.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.last_updated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.stats.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.stats.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.stats.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.stats.used.kb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.data.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.disk.fetches\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.disk.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.memory.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.quota.ram.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.quota.use.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.ops_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.item_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.quota.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.used.value.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.used.by_data.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.max_bucket_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.quota.index_memory.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.quota.memory.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.quota.total.value.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.quota.total.per_node.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.quota.used.value.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.quota.used.per_node.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.used.value.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.used.by_data.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.cmd_get\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.docs.disk_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.docs.data_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.spatial.data_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.spatial.disk_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.views.disk_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.views.data_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.cpu_utilization_rate.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.current_items.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.current_items.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.ep_bg_fetched\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.get_hits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.mcd_memory.allocated.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.mcd_memory.reserved.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.memory.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.memory.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.memory.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.ops\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.swap.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.swap.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.uptime.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.vb_replica_curr_items\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.size.root_fs\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.size.rw\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.kernel.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.kernel.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.system.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.system.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.user.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.user.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.total.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.diskio.reads\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.diskio.writes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.diskio.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.failingstreak\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.event.end_date\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.event.start_date\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.event.output\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.event.exit_code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.id.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.id.parent\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.size.virtual\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.size.regular\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.containers.paused\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.containers.running\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.containers.stopped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.containers.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.images\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.fail.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.rss.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.rss.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.usage.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.usage.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.usage.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.interface\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.in.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.in.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.in.packets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.out.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.out.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.out.packets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.cluster.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.jvm.memory.heap_init.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.jvm.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.docs.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.docs.deleted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.segments.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.segments.memory.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.store.size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.old.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.old.peak.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.old.peak_max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.old.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.young.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.young.peak.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.young.peak_max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.young.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.survivor.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.survivor.peak.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.survivor.peak_max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.survivor.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.gc.collectors.old.collection.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.gc.collectors.young.collection.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.expvar.cmdline\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.cmdline\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.total_pause.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.total_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.next_gc_limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.cpu_fraction\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.pause.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.pause.sum.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.pause.max.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.pause.avg.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.system.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.system.optained\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.system.stack\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.system.released\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.mallocs\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.frees\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.allocated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.idle\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"graphite.server.example\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.processes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.process_num\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.run_queue\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.tasks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.uptime.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.memory.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ulimit_n\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.compress.bps.in\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.compress.bps.out\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.compress.bps.rate_limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.rate.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.ssl.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.ssl.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.ssl.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.hard_max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.requests.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.sockets.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.requests.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.pipes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.pipes.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.pipes.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.session.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.session.rate.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.session.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.rate.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.frontend.key_rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.frontend.key_rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.frontend.session_reuse.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.backend.key_rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.backend.key_rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.cached_lookups\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.cache_misses\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.zlib_mem_usage.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.zlib_mem_usage.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.idle.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.weight\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.downtime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.component_type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.process_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.service_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.last_change\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.throttle.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.selected.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.tracked.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.connection.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.connection.retried\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.connection.time.avg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.denied\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.queued.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.queued.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.redispatched\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.connection.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.time.avg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.denied\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.1xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.2xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.3xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.4xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.5xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.other\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.rate.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.duration\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.health.last\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.health.fail\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.agent.last\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.failed\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.down\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.client.aborted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.server.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.server.aborted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.server.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.server.backup\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.compressor.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.compressor.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.compressor.bypassed.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.compressor.response.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.proxy.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.proxy.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.queue.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.queue.time.avg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.request.header\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.request.method\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.request.body\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.response.header\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.response.status_code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.response.body\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.broker.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.broker.address\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.topic\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.partition\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.offset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"kafka.consumergroup.meta\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.client.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.client.host\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.client.member_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.offset.newest\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.offset.oldest\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.leader\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.isr\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.replica\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.insync_replica\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.topic.error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.topic.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.broker.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.broker.address\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.uuid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.version.number\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.status.overall.state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.metrics.concurrent_connections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.metrics.requests.disconnects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.metrics.requests.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.cpu.usage.core.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.cpu.usage.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.inodes.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.inodes.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.inodes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.workingset.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.pagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.majorpagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.rootfs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.rootfs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.rootfs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.rootfs.inodes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.message\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.reason\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.timestamp.created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.namespace\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.resource_version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.uid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.self_link\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.api_version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.kind\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.resource_version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.uid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.cpu.usage.core.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.cpu.usage.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.workingset.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.pagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.majorpagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.network.rx.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.network.rx.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.network.tx.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.network.tx.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.inodes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.inodes.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.inodes.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.runtime.imagefs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.runtime.imagefs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.runtime.imagefs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.network.rx.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.network.rx.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.network.tx.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.network.tx.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.status.phase\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.status.ready\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.status.restarts\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.cpu.limit.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.cpu.request.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.request.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.paused\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.replicas.desired\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.replicas.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.replicas.unavailable\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.replicas.updated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.status.ready\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.status.unschedulable\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.cpu.allocatable.cores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.cpu.capacity.cores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.allocatable.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.pod.allocatable.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.pod.capacity.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.ip\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.host_ip\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.status.phase\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.status.ready\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.status.scheduled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.desired\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.ready\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.observed\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.labeled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.container\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.cpu.usage.core.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.cpu.usage.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.workingset.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.pagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.majorpagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.inodes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.inodes.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.inodes.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.uptime.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.threads\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.connections.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.connections.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.get.hits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.get.misses\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.cmd.get\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.cmd.set\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.read.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.written.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.items.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.items.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.evictions\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.avg_obj_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.collections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.data_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.db\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.file_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.index_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.indexes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.num_extents\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.storage_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.ns_size_mb.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.data_file_version.major\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.data_file_version.minor\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.extent_free_list.num\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.extent_free_list.size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.uptime.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.local_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.regular\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.warning\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.msg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.rollovers\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.flushes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.total.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.average.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.last.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.last_finished\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.connections.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.connections.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.connections.total_created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.commits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.journaled.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.write_to_data_files.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.compression\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.commits_in_write_lock\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.early_commits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.dt.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.prep_log_buffer.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.write_to_journal.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.write_to_data_files.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.remap_private_view.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.commits.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.commits_in_write_lock.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.extra_info.heap_usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.extra_info.page_faults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.network.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.network.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.network.requests\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.insert\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.query\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.update\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.delete\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.getmore\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.insert\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.query\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.update\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.delete\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.getmore\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.bits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.resident.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.virtual.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.mapped.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.mapped_with_journal.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.write_backs_queued\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.storage_engine.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.write.out\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.write.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.write.total_tickets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.read.out\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.read.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.read.total_tickets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.maximum.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.dirty.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.pages.read\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.pages.write\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.pages.evicted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.write.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.max_file_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.flushes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.writes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.scans\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.syncs\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.aborted.clients\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.aborted.connects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.binlog.cache.disk_use\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.binlog.cache.use\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.bytes.received\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.bytes.sent\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.threads.cached\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.threads.created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.threads.connected\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.threads.running\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.connections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.created.tmp.disk_tables\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.created.tmp.files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.created.tmp.tables\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.delayed.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.delayed.insert_threads\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.delayed.writes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.flush_commands\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.max_used_connections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.open.files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.open.streams\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.open.tables\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.opened_tables\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.command.delete\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.command.insert\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.command.select\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.command.update\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.accepts\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.handled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.requests\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.reading\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.writing\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.waiting\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.connections.accepted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.connections.queued\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.processes.idle\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.processes.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.slow_requests\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.database.oid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.database.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.user.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.user.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.application_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.client.address\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.client.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.client.port\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.backend_start\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.transaction_start\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.query_start\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.state_change\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.waiting\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.query\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.checkpoints.scheduled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.checkpoints.requested\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.checkpoints.times.write.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.checkpoints.times.sync.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.checkpoints\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.clean\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.clean_full\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.backend\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.backend_fsync\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.allocated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.stats_reset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.oid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.number_of_backends\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.transactions.commit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.transactions.rollback\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.blocks.read\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.blocks.hit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.blocks.time.read.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.blocks.time.write.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.returned\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.fetched\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.inserted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.updated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.deleted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.conflicts\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.temporary.files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.temporary.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.deadlocks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.stats_reset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"prometheus.stats.notifications.queue_length\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"prometheus.stats.notifications.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"prometheus.stats.processes.open_fds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"prometheus.stats.storage.chunks_to_persist\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.disk.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.disk.free.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.fd.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.fd.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.gc.num.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.gc.reclaimed.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.file_handle.open_attempt.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.file_handle.open_attempt.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.read.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.read.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.read.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.reopen.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.seek.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.seek.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.sync.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.sync.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.write.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.write.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.mem.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.mem.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.mnesia.disk.tx.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.mnesia.ram.tx.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.msg.store_read.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.msg.store_write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.proc.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.proc.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.processors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.queue.index.journal_write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.queue.index.read.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.queue.index.write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.run.queue\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.socket.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.socket.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.uptime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.vhost\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.durable\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.auto_delete\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.exclusive\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.node\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.arguments.max_priority\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.consumers.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.consumers.utilisation.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.messages.total.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.messages.ready.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.messages.unacknowledged.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.messages.persistent.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.memory.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.disk.reads.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.disk.writes.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.clients.connected\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.clients.longest_output_list\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.clients.biggest_input_buf\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.clients.blocked\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cluster.enabled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cpu.used.sys\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cpu.used.sys_children\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cpu.used.user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cpu.used.user_children\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.used.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.used.rss\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.used.peak\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.used.lua\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.allocator\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.loading\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.last_save.changes_since\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.bgsave.in_progress\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.last_save.time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.bgsave.last_status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.bgsave.last_time.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.bgsave.current_time.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.enabled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.rewrite.in_progress\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.rewrite.scheduled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.rewrite.last_time.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.rewrite.current_time.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.bgrewrite.last_status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.write.last_status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.role\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.connected_slaves\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.master_offset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.backlog.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.backlog.size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.backlog.first_byte_offset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.backlog.histlen\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.git_sha1\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.git_dirty\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.build_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.mode\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.os\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.arch_bits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.multiplexing_api\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.gcc_version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.process_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.run_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.tcp_port\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.uptime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.hz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.lru_clock\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.config_file\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.connections.received\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.connections.rejected\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.commands_processed\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.net.input.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.net.output.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.instantaneous.ops_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.instantaneous.input_kbps\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.instantaneous.output_kbps\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.sync.full\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.sync.partial.ok\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.sync.partial.err\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.keys.expired\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.keys.evicted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.keyspace.hits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.keyspace.misses\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.pubsub.channels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.pubsub.patterns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.latest_fork_usec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.migrate_cached_sockets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.keyspace.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.keyspace.avg_ttl\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.keyspace.keys\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.keyspace.expires\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.user.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.user.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.system.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.system.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.nice.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.nice.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.idle.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.idle.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.iowait.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.iowait.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.irq.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.irq.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.softirq.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.softirq.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.steal.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.steal.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.cores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.user.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.system.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.nice.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.idle.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.iowait.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.irq.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.softirq.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.steal.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.total.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.user.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.system.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.nice.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.idle.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.iowait.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.irq.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.softirq.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.steal.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.total.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.user.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.system.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.nice.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.idle.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.iowait.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.irq.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.softirq.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.steal.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.serial_number\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.read.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.read.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.write.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.read.time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.write.time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.io.time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.read.request.merges_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.write.request.merges_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.read.request.per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.write.request.per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.read.per_sec.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.write.per_sec.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.request.avg_size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.queue.avg_size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.await\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.service_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.busy\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.device_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.mount_point\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.free_files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.total_files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.total_size.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.total_size.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.total_size.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.1\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.5\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.15\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.norm.1\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.norm.5\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.norm.15\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.cores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.actual.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.actual.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.actual.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.swap.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.swap.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.swap.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.swap.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.out.packets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.in.packets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.in.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.out.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.in.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.out.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.ppid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.pgid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cmdline\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.username\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cwd\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.env\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.total.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.total.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.system\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.total.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.memory.size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.memory.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.memory.rss.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.memory.share\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.fd.open\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.fd.limit.soft\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.fd.limit.hard\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.cfs.period.us\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.cfs.quota.us\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.cfs.shares\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.rt.period.us\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.rt.runtime.us\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.stats.periods\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.stats.throttled.periods\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.stats.throttled.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.total.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.stats.user.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.stats.system.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.percpu\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.mem.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.mem.usage.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.mem.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.mem.failures\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.memsw.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.memsw.usage.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.memsw.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.memsw.failures\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem.usage.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem.failures\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem_tcp.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem_tcp.usage.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem_tcp.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem_tcp.failures\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.active_anon.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.active_file.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.cache.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.hierarchical_memory_limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.hierarchical_memsw_limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.inactive_anon.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.inactive_file.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.mapped_file.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.page_faults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.major_page_faults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.pages_in\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.pages_out\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.rss_huge.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.swap.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.unevictable.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.blkio.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.blkio.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.blkio.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.blkio.total.ios\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.running\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.idle\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.sleeping\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.stopped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.zombie\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.unknown\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.direction\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.family\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.local.ip\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.local.port\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.ip\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.port\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.host\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.etld_plus_one\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.host_error\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.process.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.process.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.process.cmdline\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.process.exe\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.user.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.user.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.uptime.duration.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.datacenter\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.fstype\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.capacity.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.capacity.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.capacity.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.capacity.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.datacenter\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.cpu.used.mhz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.cpu.total.mhz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.cpu.free.mhz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.memory.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.memory.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.memory.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.datacenter\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.cpu.used.mhz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.memory.used.guest.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.memory.used.host.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.memory.total.guest.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.memory.free.guest.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.custom_fields\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.approximate_data_size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.latency.avg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.ephemerals_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.followers\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.max_file_descriptor_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.latency.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.latency.min\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.num_alive_connections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.open_file_descriptor_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.outstanding_requests\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.packets.received\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.packets.sent\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.pending_syncs\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.server_state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.synced_followers\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.watch_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.znode_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_id\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"_type\", \"searchable\": true, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_index\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_score\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"number\", \"scripted\": false}]", + "fieldFormatMap": "{\"ceph.cluster_status.pg.data_bytes\": {\"id\": \"bytes\"}, \"system.filesystem.used.pct\": {\"id\": \"percent\"}, \"ceph.cluster_status.degraded.ratio\": {\"id\": \"percent\"}, \"system.cpu.nice.norm.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.mapped_file.bytes\": {\"id\": \"bytes\"}, \"system.cpu.nice.pct\": {\"id\": \"percent\"}, \"system.process.memory.share\": {\"id\": \"bytes\"}, \"couchbase.node.couch.docs.disk_size.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.logs.used.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.memory.allocatable.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.rootfs.used.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.device.used.bytes\": {\"id\": \"bytes\"}, \"system.core.nice.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.cache.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.usage.bytes\": {\"id\": \"bytes\"}, \"kubernetes.pod.network.tx.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.memory.rss.bytes\": {\"id\": \"bytes\"}, \"system.memory.used.pct\": {\"id\": \"percent\"}, \"ceph.cluster_disk.used.bytes\": {\"id\": \"bytes\"}, \"docker.cpu.user.pct\": {\"id\": \"percentage\"}, \"ceph.cluster_disk.available.bytes\": {\"id\": \"bytes\"}, \"couchbase.node.mcd_memory.allocated.bytes\": {\"id\": \"bytes\"}, \"system.memory.actual.used.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.fs.used.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.rootfs.capacity.bytes\": {\"id\": \"bytes\"}, \"system.cpu.irq.pct\": {\"id\": \"percent\"}, \"kubernetes.container.logs.capacity.bytes\": {\"id\": \"bytes\"}, \"couchbase.bucket.quota.ram.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.old.used.bytes\": {\"id\": \"bytes\"}, \"vsphere.virtualmachine.memory.total.guest.bytes\": {\"id\": \"bytes\"}, \"system.process.cpu.total.pct\": {\"id\": \"percent\"}, \"kubernetes.node.fs.capacity.bytes\": {\"id\": \"bytes\"}, \"vsphere.host.memory.used.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.jvm.memory.heap_init.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.memory.used.sindex.bytes\": {\"id\": \"bytes\"}, \"mongodb.dbstats.extent_free_list.size.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.cache.used.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.log.max_file_size.bytes\": {\"id\": \"bytes\"}, \"system.core.idle.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.active_anon.bytes\": {\"id\": \"bytes\"}, \"system.core.user.pct\": {\"id\": \"percent\"}, \"kubernetes.volume.fs.available.bytes\": {\"id\": \"bytes\"}, \"system.network.out.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.network.in.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.io.read.bytes\": {\"id\": \"bytes\"}, \"system.process.cpu.total.norm.pct\": {\"id\": \"percent\"}, \"system.network.in.bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.throttle.pct\": {\"id\": \"percentage\"}, \"vsphere.datastore.capacity.used.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.indices.segments.memory.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.young.used.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.device.free.pct\": {\"id\": \"percent\"}, \"system.cpu.system.norm.pct\": {\"id\": \"percent\"}, \"haproxy.stat.compressor.out.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.hdd.quota.total.bytes\": {\"id\": \"bytes\"}, \"system.memory.total\": {\"id\": \"bytes\"}, \"system.memory.free\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.active_file.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.log.write.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem.limit.bytes\": {\"id\": \"bytes\"}, \"system.filesystem.total\": {\"id\": \"bytes\"}, \"redis.info.memory.used.lua\": {\"id\": \"bytes\"}, \"system.memory.swap.free\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem.usage.max.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.memory.used.index.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.cache.maximum.bytes\": {\"id\": \"bytes\"}, \"vsphere.host.memory.total.bytes\": {\"id\": \"bytes\"}, \"ceph.monitor_health.store_stats.misc.bytes\": {\"id\": \"bytes\"}, \"mongodb.dbstats.file_size.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.workingset.bytes\": {\"id\": \"bytes\"}, \"system.fsstat.total_size.total\": {\"id\": \"bytes\"}, \"kubernetes.node.runtime.imagefs.used.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.hdd.free.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.device.available.pct\": {\"id\": \"percent\"}, \"vsphere.host.memory.free.bytes\": {\"id\": \"bytes\"}, \"ceph.monitor_health.store_stats.sst.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.rss.bytes\": {\"id\": \"bytes\"}, \"golang.heap.allocations.total\": {\"id\": \"bytes\"}, \"golang.heap.system.stack\": {\"id\": \"bytes\"}, \"@timestamp\": {\"id\": \"date\"}, \"kubernetes.node.runtime.imagefs.available.bytes\": {\"id\": \"bytes\"}, \"ceph.cluster_status.misplace.ratio\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.inactive_anon.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.used.by_data.bytes\": {\"id\": \"bytes\"}, \"system.cpu.idle.pct\": {\"id\": \"percent\"}, \"kubernetes.node.memory.workingset.bytes\": {\"id\": \"bytes\"}, \"system.cpu.steal.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.inactive_file.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem.usage.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.old.peak.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.old.peak_max.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.rss.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.memsw.limit.bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.compressor.bypassed.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.memory.usage.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.quota.total.per_node.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.memory.available.bytes\": {\"id\": \"bytes\"}, \"haproxy.info.memory.max.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.queue.memory.bytes\": {\"id\": \"bytes\"}, \"ceph.cluster_status.traffic.read_bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.in.bytes\": {\"id\": \"bytes\"}, \"vsphere.virtualmachine.memory.free.guest.bytes\": {\"id\": \"bytes\"}, \"couchbase.bucket.memory.used.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.survivor.used.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem_tcp.usage.bytes\": {\"id\": \"bytes\"}, \"golang.heap.allocations.active\": {\"id\": \"bytes\"}, \"system.cpu.steal.norm.pct\": {\"id\": \"percent\"}, \"kubernetes.node.fs.available.bytes\": {\"id\": \"bytes\"}, \"vsphere.datastore.capacity.free.bytes\": {\"id\": \"bytes\"}, \"system.core.iowait.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.hierarchical_memory_limit.bytes\": {\"id\": \"bytes\"}, \"system.cpu.irq.norm.pct\": {\"id\": \"percent\"}, \"vsphere.datastore.capacity.total.bytes\": {\"id\": \"bytes\"}, \"system.memory.used.bytes\": {\"id\": \"bytes\"}, \"system.cpu.total.norm.pct\": {\"id\": \"percent\"}, \"couchbase.cluster.ram.quota.used.value.bytes\": {\"id\": \"bytes\"}, \"system.diskio.read.bytes\": {\"id\": \"bytes\"}, \"system.memory.swap.used.bytes\": {\"id\": \"bytes\"}, \"vsphere.datastore.capacity.used.pct\": {\"id\": \"percent\"}, \"vsphere.virtualmachine.memory.used.host.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.limit.bytes\": {\"id\": \"bytes\"}, \"docker.memory.limit\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.available.bytes\": {\"id\": \"bytes\"}, \"mongodb.dbstats.storage_size.bytes\": {\"id\": \"bytes\"}, \"couchbase.bucket.disk.used.bytes\": {\"id\": \"bytes\"}, \"system.cpu.softirq.norm.pct\": {\"id\": \"percent\"}, \"ceph.cluster_status.pg.total_bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.mem.limit.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.hdd.total.bytes\": {\"id\": \"bytes\"}, \"docker.memory.rss.total\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.quota.total.value.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.network.out.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.survivor.peak.bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.compressor.in.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.old.max.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.used.value.bytes\": {\"id\": \"bytes\"}, \"system.cpu.user.pct\": {\"id\": \"percent\"}, \"golang.heap.system.released\": {\"id\": \"bytes\"}, \"ceph.cluster_disk.total.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.runtime.imagefs.capacity.bytes\": {\"id\": \"bytes\"}, \"system.uptime.duration.ms\": {\"params\": {\"inputFormat\": \"milliseconds\"}, \"id\": \"duration\"}, \"kubernetes.node.memory.capacity.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.disk.free.limit.bytes\": {\"id\": \"bytes\"}, \"system.core.system.pct\": {\"id\": \"percent\"}, \"couchbase.cluster.hdd.used.by_data.bytes\": {\"id\": \"bytes\"}, \"system.cpu.iowait.norm.pct\": {\"id\": \"percent\"}, \"system.cpu.system.pct\": {\"id\": \"percent\"}, \"docker.network.in.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.hierarchical_memsw_limit.bytes\": {\"id\": \"bytes\"}, \"system.filesystem.free\": {\"id\": \"bytes\"}, \"system.memory.swap.used.pct\": {\"id\": \"percent\"}, \"ceph.monitor_health.store_stats.total.bytes\": {\"id\": \"bytes\"}, \"system.core.irq.pct\": {\"id\": \"percent\"}, \"system.cpu.softirq.pct\": {\"id\": \"percent\"}, \"haproxy.stat.compressor.response.bytes\": {\"id\": \"bytes\"}, \"system.cpu.idle.norm.pct\": {\"id\": \"percent\"}, \"system.process.memory.rss.pct\": {\"id\": \"percent\"}, \"kubernetes.system.memory.workingset.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.request.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.hdd.used.value.bytes\": {\"id\": \"bytes\"}, \"kubernetes.system.memory.usage.bytes\": {\"id\": \"bytes\"}, \"ceph.monitor_health.store_stats.log.bytes\": {\"id\": \"bytes\"}, \"kubernetes.volume.fs.capacity.bytes\": {\"id\": \"bytes\"}, \"vsphere.virtualmachine.memory.used.guest.bytes\": {\"id\": \"bytes\"}, \"couchbase.node.couch.docs.data_size.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.log.size.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.gc.reclaimed.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.mem.usage.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.survivor.max.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.quota.used.per_node.bytes\": {\"id\": \"bytes\"}, \"golang.heap.allocations.allocated\": {\"id\": \"bytes\"}, \"system.core.steal.pct\": {\"id\": \"percent\"}, \"golang.heap.gc.next_gc_limit\": {\"id\": \"bytes\"}, \"rabbitmq.queue.consumers.utilisation.pct\": {\"id\": \"percentage\"}, \"docker.cpu.total.pct\": {\"id\": \"percentage\"}, \"docker.memory.usage.pct\": {\"id\": \"percentage\"}, \"mysql.status.bytes.received\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem_tcp.usage.max.bytes\": {\"id\": \"bytes\"}, \"redis.info.memory.used.peak\": {\"id\": \"bytes\"}, \"aerospike.namespace.memory.free.pct\": {\"id\": \"percent\"}, \"kubernetes.volume.fs.used.bytes\": {\"id\": \"bytes\"}, \"system.cpu.iowait.pct\": {\"id\": \"percent\"}, \"couchbase.bucket.data.used.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.cache.dirty.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.rss_huge.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.memsw.usage.bytes\": {\"id\": \"bytes\"}, \"kubernetes.pod.network.rx.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.blkio.total.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.disk.free.bytes\": {\"id\": \"bytes\"}, \"system.filesystem.used.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.network.rx.bytes\": {\"id\": \"bytes\"}, \"system.diskio.iostat.read.per_sec.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.mem.usage.max.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.young.peak_max.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.memsw.usage.max.bytes\": {\"id\": \"bytes\"}, \"system.diskio.write.bytes\": {\"id\": \"bytes\"}, \"ceph.cluster_status.pg.used_bytes\": {\"id\": \"bytes\"}, \"haproxy.info.idle.pct\": {\"id\": \"percent\"}, \"system.process.memory.size\": {\"id\": \"bytes\"}, \"kubernetes.system.memory.rss.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem_tcp.limit.bytes\": {\"id\": \"bytes\"}, \"golang.heap.allocations.idle\": {\"id\": \"bytes\"}, \"docker.network.out.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.extra_info.heap_usage.bytes\": {\"id\": \"bytes\"}, \"system.cpu.total.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.swap.bytes\": {\"id\": \"bytes\"}, \"golang.heap.system.total\": {\"id\": \"bytes\"}, \"system.fsstat.total_size.free\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.young.max.bytes\": {\"id\": \"bytes\"}, \"docker.memory.usage.total\": {\"id\": \"bytes\"}, \"mongodb.dbstats.avg_obj_size.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.total.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.logs.available.bytes\": {\"id\": \"bytes\"}, \"system.diskio.iostat.write.per_sec.bytes\": {\"id\": \"bytes\"}, \"system.filesystem.available\": {\"id\": \"bytes\"}, \"aerospike.namespace.memory.used.data.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.unevictable.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.network.tx.bytes\": {\"id\": \"bytes\"}, \"ceph.pool_disk.stats.used.bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.out.bytes\": {\"id\": \"bytes\"}, \"system.memory.actual.free\": {\"id\": \"bytes\"}, \"system.cpu.user.norm.pct\": {\"id\": \"percent\"}, \"system.process.memory.rss.bytes\": {\"id\": \"bytes\"}, \"ceph.pool_disk.stats.available.bytes\": {\"id\": \"bytes\"}, \"system.fsstat.total_size.used\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.mem.limit.bytes\": {\"id\": \"bytes\"}, \"docker.cpu.kernel.pct\": {\"id\": \"percentage\"}, \"system.memory.swap.total\": {\"id\": \"bytes\"}, \"ceph.cluster_status.traffic.write_bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.io.write.bytes\": {\"id\": \"bytes\"}, \"system.memory.actual.used.pct\": {\"id\": \"percent\"}, \"mongodb.dbstats.data_size.bytes\": {\"id\": \"bytes\"}, \"couchbase.bucket.quota.use.pct\": {\"id\": \"percent\"}, \"haproxy.info.ssl.frontend.session_reuse.pct\": {\"id\": \"percent\"}, \"mysql.status.bytes.sent\": {\"id\": \"bytes\"}, \"docker.memory.rss.pct\": {\"id\": \"percentage\"}, \"aerospike.namespace.memory.used.total.bytes\": {\"id\": \"bytes\"}, \"ceph.cluster_status.pg.avail_bytes\": {\"id\": \"bytes\"}, \"redis.info.memory.used.value\": {\"id\": \"bytes Used memory.\"}, \"system.core.softirq.pct\": {\"id\": \"percent\"}, \"redis.info.memory.used.rss\": {\"id\": \"bytes\"}, \"kubernetes.container.rootfs.available.bytes\": {\"id\": \"bytes\"}, \"docker.cpu.system.pct\": {\"id\": \"percentage\"}, \"elasticsearch.node.stats.jvm.mem.pools.young.peak.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.survivor.peak_max.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.device.total.bytes\": {\"id\": \"bytes\"}, \"docker.memory.usage.max\": {\"id\": \"bytes\"}, \"mongodb.dbstats.index_size.bytes\": {\"id\": \"bytes\"}, \"golang.heap.system.optained\": {\"id\": \"bytes\"}}", + "timeFieldName": "@timestamp", + "title": "metricbeat-*" +} diff --git a/libbeat/kibana/testdata/metricbeat-default-old.json b/libbeat/kibana/testdata/metricbeat-default-old.json new file mode 100644 index 00000000000..8626434fa7b --- /dev/null +++ b/libbeat/kibana/testdata/metricbeat-default-old.json @@ -0,0 +1,16 @@ +{ + "version": "7.0.0-alpha1", + "objects": [ + { + "attributes": { + "fields": "[{\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.timezone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"@timestamp\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"fields\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"error.message\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"error.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.provider\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.machine_type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.availability_zone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.project_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.region\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.image\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.namespace\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.annotations\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.image\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.module\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.host\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.rtt\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"metricset.namespace\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.delete.error\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.delete.not_found\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.delete.success\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.delete.timeout\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.read.error\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.read.not_found\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.read.success\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.read.timeout\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.write.error\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.write.success\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.client.write.timeout\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.device.available.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.device.free.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.device.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.device.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.hwm_breached\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.free.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.used.data.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.used.index.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.used.sindex.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.memory.used.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.node.host\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.node.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.objects.master\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.objects.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"aerospike.namespace.stop_writes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.total_accesses\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.total_kbytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.requests_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.bytes_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.bytes_per_request\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.workers.busy\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.workers.idle\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.uptime.server_uptime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.uptime.uptime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.load\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.system\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.children_user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.cpu.children_system\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.connections.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.connections.async.writing\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.connections.async.keep_alive\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.connections.async.closing\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.load.1\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.load.5\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.load.15\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.starting_up\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.reading_request\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.sending_reply\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.keepalive\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.dns_lookup\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.closing_connection\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.logging\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.gracefully_finishing\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.idle_cleanup\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.open_slot\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.waiting_for_connection\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"apache.status.scoreboard.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_disk.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_disk.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_disk.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_health.overall_status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_health.timechecks.epoch\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_health.timechecks.round.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_health.timechecks.round.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.traffic.read_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.traffic.write_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.traffic.read_op_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.traffic.write_op_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.misplace.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.misplace.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.misplace.ratio\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.degraded.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.degraded.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.degraded.ratio\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg.data_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg.avail_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg.total_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg.used_bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg_state.state_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg_state.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.pg_state.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.full\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.nearfull\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.num_osds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.num_up_osds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.num_in_osds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.num_in_osds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.cluster_status.osd.epoch\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.available.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.health\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.available.kb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.total.kb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.used.kb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.last_updated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.log.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.misc.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.sst.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.monitor_health.store_stats.last_updated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.stats.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.stats.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.stats.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"ceph.pool_disk.stats.used.kb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.data.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.disk.fetches\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.disk.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.memory.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.quota.ram.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.quota.use.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.ops_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.bucket.item_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.quota.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.used.value.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.hdd.used.by_data.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.max_bucket_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.quota.index_memory.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.quota.memory.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.quota.total.value.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.quota.total.per_node.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.quota.used.value.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.quota.used.per_node.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.used.value.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.cluster.ram.used.by_data.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.cmd_get\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.docs.disk_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.docs.data_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.spatial.data_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.spatial.disk_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.views.disk_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.couch.views.data_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.cpu_utilization_rate.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.current_items.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.current_items.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.ep_bg_fetched\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.get_hits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.mcd_memory.allocated.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.mcd_memory.reserved.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.memory.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.memory.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.memory.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.ops\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.swap.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.swap.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.uptime.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"couchbase.node.vb_replica_curr_items\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.size.root_fs\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.size.rw\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.container.tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.kernel.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.kernel.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.system.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.system.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.user.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.user.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.cpu.total.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.diskio.reads\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.diskio.writes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.diskio.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.failingstreak\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.event.end_date\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.event.start_date\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.event.output\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.healthcheck.event.exit_code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.id.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.id.parent\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.size.virtual\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.size.regular\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.labels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.image.tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.containers.paused\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.containers.running\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.containers.stopped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.containers.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.info.images\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.fail.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.rss.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.rss.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.usage.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.usage.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.memory.usage.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.interface\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.in.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.in.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.in.packets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.out.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.out.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"docker.network.out.packets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.cluster.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.jvm.memory.heap_init.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.jvm.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.docs.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.docs.deleted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.segments.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.segments.memory.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.indices.store.size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.old.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.old.peak.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.old.peak_max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.old.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.young.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.young.peak.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.young.peak_max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.young.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.survivor.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.survivor.peak.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.survivor.peak_max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.mem.pools.survivor.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.gc.collectors.old.collection.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.gc.collectors.young.collection.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.expvar.cmdline\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.cmdline\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.total_pause.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.total_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.next_gc_limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.cpu_fraction\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.pause.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.pause.sum.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.pause.max.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.gc.pause.avg.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.system.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.system.optained\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.system.stack\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.system.released\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.mallocs\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.frees\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.allocated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.idle\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"golang.heap.allocations.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"graphite.server.example\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.processes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.process_num\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.run_queue\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.tasks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.uptime.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.memory.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ulimit_n\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.compress.bps.in\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.compress.bps.out\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.compress.bps.rate_limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.rate.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.ssl.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.ssl.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.ssl.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.connection.hard_max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.requests.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.sockets.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.requests.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.pipes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.pipes.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.pipes.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.session.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.session.rate.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.session.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.rate.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.frontend.key_rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.frontend.key_rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.frontend.session_reuse.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.backend.key_rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.backend.key_rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.cached_lookups\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.ssl.cache_misses\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.zlib_mem_usage.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.zlib_mem_usage.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.info.idle.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.weight\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.downtime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.component_type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.process_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.service_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.last_change\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.throttle.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.selected.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.tracked.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.connection.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.connection.retried\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.connection.time.avg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.denied\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.queued.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.queued.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.redispatched\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.connection.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.request.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.time.avg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.denied\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.1xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.2xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.3xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.4xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.5xx\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.response.http.other\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.rate.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.rate.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.session.rate.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.duration\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.health.last\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.health.fail\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.agent.last\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.failed\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.check.down\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.client.aborted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.server.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.server.aborted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.server.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.server.backup\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.compressor.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.compressor.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.compressor.bypassed.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.compressor.response.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.proxy.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.proxy.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.queue.limit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"haproxy.stat.queue.time.avg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.request.header\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.request.method\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.request.body\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.response.header\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.response.status_code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"http.response.body\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.broker.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.broker.address\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.topic\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.partition\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.offset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"kafka.consumergroup.meta\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.client.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.client.host\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.consumergroup.client.member_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.offset.newest\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.offset.oldest\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.leader\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.isr\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.replica\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.insync_replica\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.partition.error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.topic.error.code\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.topic.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.broker.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kafka.partition.broker.address\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.uuid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.version.number\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.status.overall.state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.metrics.concurrent_connections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.metrics.requests.disconnects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kibana.status.metrics.requests.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.cpu.usage.core.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.cpu.usage.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.inodes.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.inodes.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.logs.inodes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.workingset.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.pagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.majorpagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.rootfs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.rootfs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.rootfs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.rootfs.inodes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.message\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.reason\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.timestamp.created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.namespace\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.resource_version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.uid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.metadata.self_link\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.api_version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.kind\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.resource_version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.event.involved_object.uid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.cpu.usage.core.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.cpu.usage.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.workingset.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.pagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.majorpagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.network.rx.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.network.rx.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.network.tx.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.network.tx.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.inodes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.inodes.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.fs.inodes.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.runtime.imagefs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.runtime.imagefs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.runtime.imagefs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.network.rx.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.network.rx.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.network.tx.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.network.tx.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.status.phase\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.status.ready\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.status.restarts\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.cpu.limit.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.cpu.request.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.container.memory.request.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.paused\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.replicas.desired\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.replicas.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.replicas.unavailable\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.deployment.replicas.updated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.status.ready\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.status.unschedulable\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.cpu.allocatable.cores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.cpu.capacity.cores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.allocatable.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.memory.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.pod.allocatable.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.node.pod.capacity.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.ip\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.host_ip\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.status.phase\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.status.ready\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.pod.status.scheduled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.desired\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.ready\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.observed\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.replicaset.replicas.labeled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.container\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.cpu.usage.core.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.cpu.usage.nanocores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.workingset.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.pagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.system.memory.majorpagefaults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.capacity.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.available.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.inodes.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.inodes.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"kubernetes.volume.fs.inodes.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.uptime.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.threads\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.connections.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.connections.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.get.hits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.get.misses\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.cmd.get\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.cmd.set\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.read.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.written.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.items.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.items.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"memcached.stats.evictions\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.avg_obj_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.collections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.data_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.db\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.file_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.index_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.indexes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.num_extents\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.objects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.storage_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.ns_size_mb.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.data_file_version.major\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.data_file_version.minor\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.extent_free_list.num\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.dbstats.extent_free_list.size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.uptime.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.local_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.regular\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.warning\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.msg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.asserts.rollovers\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.flushes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.total.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.average.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.last.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.background_flushing.last_finished\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.connections.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.connections.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.connections.total_created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.commits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.journaled.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.write_to_data_files.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.compression\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.commits_in_write_lock\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.early_commits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.dt.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.prep_log_buffer.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.write_to_journal.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.write_to_data_files.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.remap_private_view.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.commits.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.journaling.times.commits_in_write_lock.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.extra_info.heap_usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.extra_info.page_faults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.network.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.network.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.network.requests\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.insert\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.query\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.update\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.delete\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.getmore\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.insert\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.query\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.update\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.delete\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.getmore\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.opcounters_replicated.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.bits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.resident.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.virtual.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.mapped.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.memory.mapped_with_journal.mb\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.write_backs_queued\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.storage_engine.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.write.out\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.write.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.write.total_tickets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.read.out\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.read.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.concurrent_transactions.read.total_tickets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.maximum.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.dirty.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.pages.read\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.pages.write\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.cache.pages.evicted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.write.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.max_file_size.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.flushes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.writes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.scans\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mongodb.status.wired_tiger.log.syncs\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.aborted.clients\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.aborted.connects\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.binlog.cache.disk_use\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.binlog.cache.use\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.bytes.received\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.bytes.sent\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.threads.cached\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.threads.created\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.threads.connected\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.threads.running\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.connections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.created.tmp.disk_tables\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.created.tmp.files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.created.tmp.tables\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.delayed.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.delayed.insert_threads\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.delayed.writes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.flush_commands\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.max_used_connections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.open.files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.open.streams\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.open.tables\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.opened_tables\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.command.delete\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.command.insert\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.command.select\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"mysql.status.command.update\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.accepts\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.handled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.requests\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.current\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.reading\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.writing\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"nginx.stubstatus.waiting\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.connections.accepted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.connections.queued\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.processes.idle\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.processes.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"php_fpm.pool.slow_requests\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.database.oid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.database.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.user.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.user.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.application_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.client.address\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.client.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.client.port\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.backend_start\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.transaction_start\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.query_start\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.state_change\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.waiting\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.activity.query\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.checkpoints.scheduled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.checkpoints.requested\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.checkpoints.times.write.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.checkpoints.times.sync.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.checkpoints\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.clean\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.clean_full\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.backend\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.backend_fsync\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.buffers.allocated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.bgwriter.stats_reset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.oid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.number_of_backends\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.transactions.commit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.transactions.rollback\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.blocks.read\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.blocks.hit\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.blocks.time.read.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.blocks.time.write.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.returned\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.fetched\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.inserted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.updated\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.rows.deleted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.conflicts\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.temporary.files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.temporary.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.deadlocks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"postgresql.database.stats_reset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"prometheus.stats.notifications.queue_length\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"prometheus.stats.notifications.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"prometheus.stats.processes.open_fds\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"prometheus.stats.storage.chunks_to_persist\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.disk.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.disk.free.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.fd.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.fd.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.gc.num.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.gc.reclaimed.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.file_handle.open_attempt.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.file_handle.open_attempt.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.read.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.read.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.read.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.reopen.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.seek.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.seek.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.sync.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.sync.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.write.avg.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.write.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.io.write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.mem.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.mem.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.mnesia.disk.tx.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.mnesia.ram.tx.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.msg.store_read.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.msg.store_write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.proc.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.proc.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.processors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.queue.index.journal_write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.queue.index.read.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.queue.index.write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.run.queue\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.socket.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.socket.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.node.uptime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.vhost\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.durable\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.auto_delete\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.exclusive\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.node\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.arguments.max_priority\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.consumers.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.consumers.utilisation.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.messages.total.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.messages.ready.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.messages.unacknowledged.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.messages.persistent.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.memory.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.disk.reads.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"rabbitmq.queue.disk.writes.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.clients.connected\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.clients.longest_output_list\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.clients.biggest_input_buf\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.clients.blocked\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cluster.enabled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cpu.used.sys\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cpu.used.sys_children\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cpu.used.user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.cpu.used.user_children\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.used.value\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.used.rss\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.used.peak\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.used.lua\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.memory.allocator\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.loading\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.last_save.changes_since\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.bgsave.in_progress\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.last_save.time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.bgsave.last_status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.bgsave.last_time.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.rdb.bgsave.current_time.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.enabled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.rewrite.in_progress\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.rewrite.scheduled\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.rewrite.last_time.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.rewrite.current_time.sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.bgrewrite.last_status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.persistence.aof.write.last_status\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.role\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.connected_slaves\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.master_offset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.backlog.active\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.backlog.size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.backlog.first_byte_offset\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.replication.backlog.histlen\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.git_sha1\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.git_dirty\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.build_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.mode\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.os\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.arch_bits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.multiplexing_api\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.gcc_version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.process_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.run_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.tcp_port\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.uptime\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.hz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.lru_clock\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.server.config_file\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.connections.received\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.connections.rejected\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.commands_processed\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.net.input.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.net.output.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.instantaneous.ops_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.instantaneous.input_kbps\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.instantaneous.output_kbps\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.sync.full\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.sync.partial.ok\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.sync.partial.err\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.keys.expired\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.keys.evicted\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.keyspace.hits\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.keyspace.misses\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.pubsub.channels\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.pubsub.patterns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.latest_fork_usec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.info.stats.migrate_cached_sockets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.keyspace.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.keyspace.avg_ttl\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.keyspace.keys\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"redis.keyspace.expires\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.user.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.user.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.system.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.system.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.nice.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.nice.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.idle.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.idle.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.iowait.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.iowait.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.irq.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.irq.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.softirq.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.softirq.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.steal.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.core.steal.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.cores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.user.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.system.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.nice.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.idle.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.iowait.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.irq.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.softirq.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.steal.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.total.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.user.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.system.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.nice.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.idle.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.iowait.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.irq.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.softirq.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.steal.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.total.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.user.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.system.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.nice.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.idle.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.iowait.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.irq.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.softirq.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.cpu.steal.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.serial_number\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.read.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.write.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.read.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.write.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.read.time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.write.time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.io.time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.read.request.merges_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.write.request.merges_per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.read.request.per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.write.request.per_sec\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.read.per_sec.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.write.per_sec.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.request.avg_size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.queue.avg_size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.await\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.service_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.diskio.iostat.busy\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.available\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.device_name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.mount_point\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.free_files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.filesystem.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.total_files\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.total_size.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.total_size.used\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.fsstat.total_size.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.1\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.5\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.15\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.norm.1\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.norm.5\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.norm.15\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.load.cores\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.actual.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.actual.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.actual.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.swap.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.swap.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.swap.free\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.memory.swap.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.out.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.in.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.out.packets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.in.packets\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.in.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.out.errors\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.in.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.network.out.dropped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.ppid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.pgid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cmdline\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.username\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cwd\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.env\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.user\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.total.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.total.norm.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.system\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.total.ticks\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cpu.start_time\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.memory.size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.memory.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.memory.rss.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.memory.share\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.fd.open\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.fd.limit.soft\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.fd.limit.hard\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.cfs.period.us\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.cfs.quota.us\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.cfs.shares\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.rt.period.us\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.rt.runtime.us\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.stats.periods\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.stats.throttled.periods\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpu.stats.throttled.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.total.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.stats.user.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.stats.system.ns\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.cpuacct.percpu\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.mem.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.mem.usage.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.mem.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.mem.failures\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.memsw.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.memsw.usage.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.memsw.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.memsw.failures\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem.usage.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem.failures\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem_tcp.usage.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem_tcp.usage.max.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem_tcp.limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.kmem_tcp.failures\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.active_anon.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.active_file.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.cache.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.hierarchical_memory_limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.hierarchical_memsw_limit.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.inactive_anon.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.inactive_file.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.mapped_file.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.page_faults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.major_page_faults\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.pages_in\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.pages_out\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.rss.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.rss_huge.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.swap.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.memory.stats.unevictable.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.blkio.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.blkio.path\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.blkio.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.cgroup.blkio.total.ios\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.total\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.running\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.idle\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.sleeping\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.stopped\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.zombie\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.process.summary.unknown\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.direction\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.family\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.local.ip\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.local.port\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.ip\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.port\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.host\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.etld_plus_one\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.remote.host_error\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.process.pid\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.process.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.process.cmdline\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.process.exe\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.user.id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.socket.user.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"system.uptime.duration.ms\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.datacenter\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.fstype\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.capacity.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.capacity.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.capacity.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.datastore.capacity.used.pct\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.datacenter\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.cpu.used.mhz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.cpu.total.mhz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.cpu.free.mhz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.memory.used.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.memory.total.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.host.memory.free.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.datacenter\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.cpu.used.mhz\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.memory.used.guest.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.memory.used.host.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.memory.total.guest.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.memory.free.guest.bytes\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"vsphere.virtualmachine.custom_fields\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.approximate_data_size\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.latency.avg\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.ephemerals_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.followers\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.max_file_descriptor_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.latency.max\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.latency.min\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.num_alive_connections\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.open_file_descriptor_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.outstanding_requests\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.packets.received\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.packets.sent\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.pending_syncs\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.server_state\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.synced_followers\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.watch_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"zookeeper.mntr.znode_count\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_id\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"_type\", \"searchable\": true, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_index\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": false, \"name\": \"_score\", \"searchable\": false, \"indexed\": false, \"doc_values\": false, \"type\": \"number\", \"scripted\": false}]", + "fieldFormatMap": "{\"ceph.cluster_status.pg.data_bytes\": {\"id\": \"bytes\"}, \"system.filesystem.used.pct\": {\"id\": \"percent\"}, \"ceph.cluster_status.degraded.ratio\": {\"id\": \"percent\"}, \"system.cpu.nice.norm.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.mapped_file.bytes\": {\"id\": \"bytes\"}, \"system.cpu.nice.pct\": {\"id\": \"percent\"}, \"system.process.memory.share\": {\"id\": \"bytes\"}, \"couchbase.node.couch.docs.disk_size.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.logs.used.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.memory.allocatable.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.rootfs.used.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.device.used.bytes\": {\"id\": \"bytes\"}, \"system.core.nice.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.cache.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.usage.bytes\": {\"id\": \"bytes\"}, \"kubernetes.pod.network.tx.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.memory.rss.bytes\": {\"id\": \"bytes\"}, \"system.memory.used.pct\": {\"id\": \"percent\"}, \"ceph.cluster_disk.used.bytes\": {\"id\": \"bytes\"}, \"docker.cpu.user.pct\": {\"id\": \"percentage\"}, \"ceph.cluster_disk.available.bytes\": {\"id\": \"bytes\"}, \"couchbase.node.mcd_memory.allocated.bytes\": {\"id\": \"bytes\"}, \"system.memory.actual.used.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.fs.used.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.rootfs.capacity.bytes\": {\"id\": \"bytes\"}, \"system.cpu.irq.pct\": {\"id\": \"percent\"}, \"kubernetes.container.logs.capacity.bytes\": {\"id\": \"bytes\"}, \"couchbase.bucket.quota.ram.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.old.used.bytes\": {\"id\": \"bytes\"}, \"vsphere.virtualmachine.memory.total.guest.bytes\": {\"id\": \"bytes\"}, \"system.process.cpu.total.pct\": {\"id\": \"percent\"}, \"kubernetes.node.fs.capacity.bytes\": {\"id\": \"bytes\"}, \"vsphere.host.memory.used.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.jvm.memory.heap_init.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.memory.used.sindex.bytes\": {\"id\": \"bytes\"}, \"mongodb.dbstats.extent_free_list.size.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.cache.used.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.log.max_file_size.bytes\": {\"id\": \"bytes\"}, \"system.core.idle.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.active_anon.bytes\": {\"id\": \"bytes\"}, \"system.core.user.pct\": {\"id\": \"percent\"}, \"kubernetes.volume.fs.available.bytes\": {\"id\": \"bytes\"}, \"system.network.out.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.network.in.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.io.read.bytes\": {\"id\": \"bytes\"}, \"system.process.cpu.total.norm.pct\": {\"id\": \"percent\"}, \"system.network.in.bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.throttle.pct\": {\"id\": \"percentage\"}, \"vsphere.datastore.capacity.used.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.indices.segments.memory.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.young.used.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.device.free.pct\": {\"id\": \"percent\"}, \"system.cpu.system.norm.pct\": {\"id\": \"percent\"}, \"haproxy.stat.compressor.out.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.hdd.quota.total.bytes\": {\"id\": \"bytes\"}, \"system.memory.total\": {\"id\": \"bytes\"}, \"system.memory.free\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.active_file.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.log.write.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem.limit.bytes\": {\"id\": \"bytes\"}, \"system.filesystem.total\": {\"id\": \"bytes\"}, \"redis.info.memory.used.lua\": {\"id\": \"bytes\"}, \"system.memory.swap.free\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem.usage.max.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.memory.used.index.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.cache.maximum.bytes\": {\"id\": \"bytes\"}, \"vsphere.host.memory.total.bytes\": {\"id\": \"bytes\"}, \"ceph.monitor_health.store_stats.misc.bytes\": {\"id\": \"bytes\"}, \"mongodb.dbstats.file_size.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.workingset.bytes\": {\"id\": \"bytes\"}, \"system.fsstat.total_size.total\": {\"id\": \"bytes\"}, \"kubernetes.node.runtime.imagefs.used.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.hdd.free.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.device.available.pct\": {\"id\": \"percent\"}, \"vsphere.host.memory.free.bytes\": {\"id\": \"bytes\"}, \"ceph.monitor_health.store_stats.sst.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.rss.bytes\": {\"id\": \"bytes\"}, \"golang.heap.allocations.total\": {\"id\": \"bytes\"}, \"golang.heap.system.stack\": {\"id\": \"bytes\"}, \"@timestamp\": {\"id\": \"date\"}, \"kubernetes.node.runtime.imagefs.available.bytes\": {\"id\": \"bytes\"}, \"ceph.cluster_status.misplace.ratio\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.inactive_anon.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.used.by_data.bytes\": {\"id\": \"bytes\"}, \"system.cpu.idle.pct\": {\"id\": \"percent\"}, \"kubernetes.node.memory.workingset.bytes\": {\"id\": \"bytes\"}, \"system.cpu.steal.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.inactive_file.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem.usage.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.old.peak.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.old.peak_max.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.rss.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.memsw.limit.bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.compressor.bypassed.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.memory.usage.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.quota.total.per_node.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.memory.available.bytes\": {\"id\": \"bytes\"}, \"haproxy.info.memory.max.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.queue.memory.bytes\": {\"id\": \"bytes\"}, \"ceph.cluster_status.traffic.read_bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.in.bytes\": {\"id\": \"bytes\"}, \"vsphere.virtualmachine.memory.free.guest.bytes\": {\"id\": \"bytes\"}, \"couchbase.bucket.memory.used.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.survivor.used.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem_tcp.usage.bytes\": {\"id\": \"bytes\"}, \"golang.heap.allocations.active\": {\"id\": \"bytes\"}, \"system.cpu.steal.norm.pct\": {\"id\": \"percent\"}, \"kubernetes.node.fs.available.bytes\": {\"id\": \"bytes\"}, \"vsphere.datastore.capacity.free.bytes\": {\"id\": \"bytes\"}, \"system.core.iowait.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.hierarchical_memory_limit.bytes\": {\"id\": \"bytes\"}, \"system.cpu.irq.norm.pct\": {\"id\": \"percent\"}, \"vsphere.datastore.capacity.total.bytes\": {\"id\": \"bytes\"}, \"system.memory.used.bytes\": {\"id\": \"bytes\"}, \"system.cpu.total.norm.pct\": {\"id\": \"percent\"}, \"couchbase.cluster.ram.quota.used.value.bytes\": {\"id\": \"bytes\"}, \"system.diskio.read.bytes\": {\"id\": \"bytes\"}, \"system.memory.swap.used.bytes\": {\"id\": \"bytes\"}, \"vsphere.datastore.capacity.used.pct\": {\"id\": \"percent\"}, \"vsphere.virtualmachine.memory.used.host.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.limit.bytes\": {\"id\": \"bytes\"}, \"docker.memory.limit\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.available.bytes\": {\"id\": \"bytes\"}, \"mongodb.dbstats.storage_size.bytes\": {\"id\": \"bytes\"}, \"couchbase.bucket.disk.used.bytes\": {\"id\": \"bytes\"}, \"system.cpu.softirq.norm.pct\": {\"id\": \"percent\"}, \"ceph.cluster_status.pg.total_bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.mem.limit.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.hdd.total.bytes\": {\"id\": \"bytes\"}, \"docker.memory.rss.total\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.quota.total.value.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.network.out.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.survivor.peak.bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.compressor.in.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.old.max.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.used.value.bytes\": {\"id\": \"bytes\"}, \"system.cpu.user.pct\": {\"id\": \"percent\"}, \"golang.heap.system.released\": {\"id\": \"bytes\"}, \"ceph.cluster_disk.total.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.runtime.imagefs.capacity.bytes\": {\"id\": \"bytes\"}, \"system.uptime.duration.ms\": {\"params\": {\"inputFormat\": \"milliseconds\"}, \"id\": \"duration\"}, \"kubernetes.node.memory.capacity.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.disk.free.limit.bytes\": {\"id\": \"bytes\"}, \"system.core.system.pct\": {\"id\": \"percent\"}, \"couchbase.cluster.hdd.used.by_data.bytes\": {\"id\": \"bytes\"}, \"system.cpu.iowait.norm.pct\": {\"id\": \"percent\"}, \"system.cpu.system.pct\": {\"id\": \"percent\"}, \"docker.network.in.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.hierarchical_memsw_limit.bytes\": {\"id\": \"bytes\"}, \"system.filesystem.free\": {\"id\": \"bytes\"}, \"system.memory.swap.used.pct\": {\"id\": \"percent\"}, \"ceph.monitor_health.store_stats.total.bytes\": {\"id\": \"bytes\"}, \"system.core.irq.pct\": {\"id\": \"percent\"}, \"system.cpu.softirq.pct\": {\"id\": \"percent\"}, \"haproxy.stat.compressor.response.bytes\": {\"id\": \"bytes\"}, \"system.cpu.idle.norm.pct\": {\"id\": \"percent\"}, \"system.process.memory.rss.pct\": {\"id\": \"percent\"}, \"kubernetes.system.memory.workingset.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.memory.request.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.hdd.used.value.bytes\": {\"id\": \"bytes\"}, \"kubernetes.system.memory.usage.bytes\": {\"id\": \"bytes\"}, \"ceph.monitor_health.store_stats.log.bytes\": {\"id\": \"bytes\"}, \"kubernetes.volume.fs.capacity.bytes\": {\"id\": \"bytes\"}, \"vsphere.virtualmachine.memory.used.guest.bytes\": {\"id\": \"bytes\"}, \"couchbase.node.couch.docs.data_size.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.log.size.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.gc.reclaimed.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.mem.usage.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.survivor.max.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.quota.used.per_node.bytes\": {\"id\": \"bytes\"}, \"golang.heap.allocations.allocated\": {\"id\": \"bytes\"}, \"system.core.steal.pct\": {\"id\": \"percent\"}, \"golang.heap.gc.next_gc_limit\": {\"id\": \"bytes\"}, \"rabbitmq.queue.consumers.utilisation.pct\": {\"id\": \"percentage\"}, \"docker.cpu.total.pct\": {\"id\": \"percentage\"}, \"docker.memory.usage.pct\": {\"id\": \"percentage\"}, \"mysql.status.bytes.received\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem_tcp.usage.max.bytes\": {\"id\": \"bytes\"}, \"redis.info.memory.used.peak\": {\"id\": \"bytes\"}, \"aerospike.namespace.memory.free.pct\": {\"id\": \"percent\"}, \"kubernetes.volume.fs.used.bytes\": {\"id\": \"bytes\"}, \"system.cpu.iowait.pct\": {\"id\": \"percent\"}, \"couchbase.bucket.data.used.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.wired_tiger.cache.dirty.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.rss_huge.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.memsw.usage.bytes\": {\"id\": \"bytes\"}, \"kubernetes.pod.network.rx.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.blkio.total.bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.disk.free.bytes\": {\"id\": \"bytes\"}, \"system.filesystem.used.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.network.rx.bytes\": {\"id\": \"bytes\"}, \"system.diskio.iostat.read.per_sec.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.mem.usage.max.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.young.peak_max.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.memsw.usage.max.bytes\": {\"id\": \"bytes\"}, \"system.diskio.write.bytes\": {\"id\": \"bytes\"}, \"ceph.cluster_status.pg.used_bytes\": {\"id\": \"bytes\"}, \"haproxy.info.idle.pct\": {\"id\": \"percent\"}, \"system.process.memory.size\": {\"id\": \"bytes\"}, \"kubernetes.system.memory.rss.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.kmem_tcp.limit.bytes\": {\"id\": \"bytes\"}, \"golang.heap.allocations.idle\": {\"id\": \"bytes\"}, \"docker.network.out.bytes\": {\"id\": \"bytes\"}, \"mongodb.status.extra_info.heap_usage.bytes\": {\"id\": \"bytes\"}, \"system.cpu.total.pct\": {\"id\": \"percent\"}, \"system.process.cgroup.memory.stats.swap.bytes\": {\"id\": \"bytes\"}, \"golang.heap.system.total\": {\"id\": \"bytes\"}, \"system.fsstat.total_size.free\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.young.max.bytes\": {\"id\": \"bytes\"}, \"docker.memory.usage.total\": {\"id\": \"bytes\"}, \"mongodb.dbstats.avg_obj_size.bytes\": {\"id\": \"bytes\"}, \"couchbase.cluster.ram.total.bytes\": {\"id\": \"bytes\"}, \"kubernetes.container.logs.available.bytes\": {\"id\": \"bytes\"}, \"system.diskio.iostat.write.per_sec.bytes\": {\"id\": \"bytes\"}, \"system.filesystem.available\": {\"id\": \"bytes\"}, \"aerospike.namespace.memory.used.data.bytes\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.stats.unevictable.bytes\": {\"id\": \"bytes\"}, \"kubernetes.node.network.tx.bytes\": {\"id\": \"bytes\"}, \"ceph.pool_disk.stats.used.bytes\": {\"id\": \"bytes\"}, \"haproxy.stat.out.bytes\": {\"id\": \"bytes\"}, \"system.memory.actual.free\": {\"id\": \"bytes\"}, \"system.cpu.user.norm.pct\": {\"id\": \"percent\"}, \"system.process.memory.rss.bytes\": {\"id\": \"bytes\"}, \"ceph.pool_disk.stats.available.bytes\": {\"id\": \"bytes\"}, \"system.fsstat.total_size.used\": {\"id\": \"bytes\"}, \"system.process.cgroup.memory.mem.limit.bytes\": {\"id\": \"bytes\"}, \"docker.cpu.kernel.pct\": {\"id\": \"percentage\"}, \"system.memory.swap.total\": {\"id\": \"bytes\"}, \"ceph.cluster_status.traffic.write_bytes\": {\"id\": \"bytes\"}, \"rabbitmq.node.io.write.bytes\": {\"id\": \"bytes\"}, \"system.memory.actual.used.pct\": {\"id\": \"percent\"}, \"mongodb.dbstats.data_size.bytes\": {\"id\": \"bytes\"}, \"couchbase.bucket.quota.use.pct\": {\"id\": \"percent\"}, \"haproxy.info.ssl.frontend.session_reuse.pct\": {\"id\": \"percent\"}, \"mysql.status.bytes.sent\": {\"id\": \"bytes\"}, \"docker.memory.rss.pct\": {\"id\": \"percentage\"}, \"aerospike.namespace.memory.used.total.bytes\": {\"id\": \"bytes\"}, \"ceph.cluster_status.pg.avail_bytes\": {\"id\": \"bytes\"}, \"redis.info.memory.used.value\": {\"id\": \"bytes Used memory.\"}, \"system.core.softirq.pct\": {\"id\": \"percent\"}, \"redis.info.memory.used.rss\": {\"id\": \"bytes\"}, \"kubernetes.container.rootfs.available.bytes\": {\"id\": \"bytes\"}, \"docker.cpu.system.pct\": {\"id\": \"percentage\"}, \"elasticsearch.node.stats.jvm.mem.pools.young.peak.bytes\": {\"id\": \"bytes\"}, \"elasticsearch.node.stats.jvm.mem.pools.survivor.peak_max.bytes\": {\"id\": \"bytes\"}, \"aerospike.namespace.device.total.bytes\": {\"id\": \"bytes\"}, \"docker.memory.usage.max\": {\"id\": \"bytes\"}, \"mongodb.dbstats.index_size.bytes\": {\"id\": \"bytes\"}, \"golang.heap.system.optained\": {\"id\": \"bytes\"}}", + "timeFieldName": "@timestamp", + "title": "metricbeat-*" + }, + "version": 1, + "type": "index-pattern", + "id": "metricbeat-*" + } + ] +} diff --git a/libbeat/kibana/transform.go b/libbeat/kibana/transform.go new file mode 100644 index 00000000000..46938294cd1 --- /dev/null +++ b/libbeat/kibana/transform.go @@ -0,0 +1,132 @@ +package kibana + +import ( + "errors" + "fmt" + + "github.com/elastic/beats/libbeat/common" +) + +func TransformFields(timeFieldName string, title string, commonFields common.Fields) common.MapStr { + fields := []common.MapStr{} + fieldFormatMap := common.MapStr{} + keys := common.MapStr{} + + transformFields(keys, commonFields, &fields, fieldFormatMap, "") + + // add some meta fields + truthy := true + falsy := false + add(common.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap) + add(common.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}, &fields, fieldFormatMap) + add(common.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap) + add(common.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap) + + return common.MapStr{ + "timeFieldName": timeFieldName, + "title": title, + "fields": fields, + "fieldFormatMap": fieldFormatMap, + } +} + +func transformFields(keys common.MapStr, commonFields common.Fields, fields *[]common.MapStr, fieldFormatMap common.MapStr, path string) { + for _, f := range commonFields { + f.Path = f.Name + if path != "" { + f.Path = path + "." + f.Name + } + + if keys[f.Path] != nil { + msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.", f.Path) + panic(errors.New(msg)) + } + + if f.Type == "group" { + transformFields(keys, f.Fields, fields, fieldFormatMap, f.Path) + } else { + // set default values (as done in python script) + keys[f.Path] = true + + truthy := true + falsy := false + f.Index = &truthy + f.Analyzed = &falsy + f.DocValues = &truthy + f.Searchable = &truthy + f.Aggregatable = &truthy + add(f, fields, fieldFormatMap) + } + } +} + +func add(f common.Field, fields *[]common.MapStr, fieldFormatMap common.MapStr) { + field, fieldFormat := transformField(f) + *fields = append(*fields, field) + if fieldFormat != nil { + fieldFormatMap[field["name"].(string)] = fieldFormat + } + +} + +func transformField(f common.Field) (common.MapStr, common.MapStr) { + field := common.MapStr{ + "name": f.Path, + "count": 0, + "scripted": false, + "indexed": getVal(f.Index, true), + "analyzed": getVal(f.Analyzed, false), + "doc_values": getVal(f.DocValues, true), + "searchable": getVal(f.Searchable, true), + "aggregatable": getVal(f.Aggregatable, true), + } + + if t, ok := typeMapping[f.Type]; ok == true { + field["type"] = t + } + + if f.Type == "text" { + field["aggregatable"] = false + } + + var format common.MapStr + if f.Format != "" || f.Pattern != "" { + format = common.MapStr{} + + if f.Format != "" { + format["id"] = f.Format + if f.InputFormat != "" { + format["params"] = common.MapStr{"inputFormat": f.InputFormat} + } + } + if f.Pattern != "" { + format["params"] = common.MapStr{"pattern": f.Pattern} + } + } + + return field, format +} + +func getVal(valP *bool, def bool) bool { + if valP != nil { + return *valP + } + return def +} + +var ( + typeMapping = map[string]string{ + "half_float": "number", + "scaled_float": "number", + "float": "number", + "integer": "number", + "long": "number", + "short": "number", + "byte": "number", + "text": "string", + "keyword": "string", + "": "string", + "geo_point": "geo_point", + "date": "date", + } +) diff --git a/libbeat/kibana/transform_test.go b/libbeat/kibana/transform_test.go new file mode 100644 index 00000000000..0f5f58b0ec6 --- /dev/null +++ b/libbeat/kibana/transform_test.go @@ -0,0 +1,279 @@ +package kibana + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +var ( + truthy = true + falsy = false + ctMetaData = 4 +) + +func TestEmpty(t *testing.T) { + out := TransformFields("name", "title", common.Fields{}) + expected := common.MapStr{ + "timeFieldName": "name", + "title": "title", + "fieldFormatMap": common.MapStr{}, + "fields": []common.MapStr{ + common.MapStr{ + "name": "_id", + "type": "string", + "scripted": false, + "aggregatable": false, + "analyzed": false, + "count": 0, + "indexed": false, + "doc_values": false, + "searchable": false, + }, + common.MapStr{ + "name": "_type", + "type": "string", + "scripted": false, + "count": 0, + "aggregatable": true, + "analyzed": false, + "indexed": false, + "doc_values": false, + "searchable": true, + }, + common.MapStr{ + "name": "_index", + "type": "string", + "scripted": false, + "count": 0, + "aggregatable": false, + "analyzed": false, + "indexed": false, + "doc_values": false, + "searchable": false, + }, + common.MapStr{ + "name": "_score", + "type": "number", + "scripted": false, + "count": 0, + "aggregatable": false, + "analyzed": false, + "indexed": false, + "doc_values": false, + "searchable": false, + }, + }, + } + assert.Equal(t, expected, out) +} + +func TestErrors(t *testing.T) { + commonFields := common.Fields{ + common.Field{Name: "context", Path: "something"}, + common.Field{Name: "context", Path: "something", Type: "keyword"}, + } + assert.Panics(t, func() { TransformFields("", "", commonFields) }) +} + +func TestTransformTypes(t *testing.T) { + tests := []struct { + commonField common.Field + expected interface{} + }{ + {commonField: common.Field{}, expected: "string"}, + {commonField: common.Field{Type: "half_float"}, expected: "number"}, + {commonField: common.Field{Type: "scaled_float"}, expected: "number"}, + {commonField: common.Field{Type: "float"}, expected: "number"}, + {commonField: common.Field{Type: "integer"}, expected: "number"}, + {commonField: common.Field{Type: "long"}, expected: "number"}, + {commonField: common.Field{Type: "short"}, expected: "number"}, + {commonField: common.Field{Type: "byte"}, expected: "number"}, + {commonField: common.Field{Type: "keyword"}, expected: "string"}, + {commonField: common.Field{Type: "text"}, expected: "string"}, + {commonField: common.Field{Type: "string"}, expected: nil}, + {commonField: common.Field{Type: "date"}, expected: "date"}, + {commonField: common.Field{Type: "geo_point"}, expected: "geo_point"}, + {commonField: common.Field{Type: "invalid"}, expected: nil}, + } + for idx, test := range tests { + out := TransformFields("", "", common.Fields{test.commonField})["fields"].([]common.MapStr)[0] + assert.Equal(t, test.expected, out["type"], fmt.Sprintf("Failed for idx %v", idx)) + } +} + +func TestTransformGroup(t *testing.T) { + tests := []struct { + commonFields common.Fields + expected []string + }{ + { + commonFields: common.Fields{common.Field{Name: "context", Path: "something"}}, + expected: []string{"context"}, + }, + { + commonFields: common.Fields{ + common.Field{ + Name: "context", + Type: "group", + Fields: common.Fields{ + common.Field{Name: "another", Type: ""}, + }, + }, + common.Field{ + Name: "context", + Type: "group", + Fields: common.Fields{ + common.Field{Name: "type", Type: ""}, + common.Field{ + Name: "metric", + Type: "group", + Fields: common.Fields{ + common.Field{Name: "object"}, + }, + }, + }, + }, + }, + expected: []string{"context.another", "context.type", "context.metric.object"}, + }, + } + for idx, test := range tests { + out := TransformFields("", "", test.commonFields)["fields"].([]common.MapStr) + assert.Equal(t, len(test.expected)+ctMetaData, len(out)) + for i, e := range test.expected { + assert.Equal(t, e, out[i]["name"], fmt.Sprintf("Failed for idx %v", idx)) + } + } +} + +func TestTransformMisc(t *testing.T) { + tests := []struct { + commonField common.Field + expected interface{} + attr string + }{ + {commonField: common.Field{}, expected: 0, attr: "count"}, + + // searchable always set to true except for meta fields + {commonField: common.Field{}, expected: true, attr: "searchable"}, + {commonField: common.Field{Searchable: &truthy}, expected: true, attr: "searchable"}, + {commonField: common.Field{Searchable: &falsy}, expected: true, attr: "searchable"}, + + // aggregatable always set to true except for meta fields or type text + {commonField: common.Field{}, expected: true, attr: "aggregatable"}, + {commonField: common.Field{Aggregatable: &truthy}, expected: true, attr: "aggregatable"}, + {commonField: common.Field{Aggregatable: &falsy}, expected: true, attr: "aggregatable"}, + {commonField: common.Field{Type: "keyword"}, expected: true, attr: "aggregatable"}, + {commonField: common.Field{Aggregatable: &truthy, Type: "text"}, expected: false, attr: "aggregatable"}, + {commonField: common.Field{Type: "text"}, expected: false, attr: "aggregatable"}, + + // analyzed always set to false except for meta fields + {commonField: common.Field{}, expected: false, attr: "analyzed"}, + {commonField: common.Field{Analyzed: &truthy}, expected: false, attr: "analyzed"}, + {commonField: common.Field{Analyzed: &falsy}, expected: false, attr: "analyzed"}, + + // indexed always set to true except for meta fields + {commonField: common.Field{}, expected: true, attr: "indexed"}, + {commonField: common.Field{Index: &truthy}, expected: true, attr: "indexed"}, + {commonField: common.Field{Index: &falsy}, expected: true, attr: "indexed"}, + + // doc_values always set to true except for meta fields + {commonField: common.Field{}, expected: true, attr: "doc_values"}, + {commonField: common.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, + {commonField: common.Field{DocValues: &falsy}, expected: true, attr: "doc_values"}, + + // scripted always set to false + {commonField: common.Field{}, expected: false, attr: "scripted"}, + } + for idx, test := range tests { + out := TransformFields("", "", common.Fields{test.commonField})["fields"].([]common.MapStr)[0] + msg := fmt.Sprintf("(%v): expected '%s' to be <%v> but was <%v>", idx, test.attr, test.expected, out[test.attr]) + assert.Equal(t, test.expected, out[test.attr], msg) + } +} + +func TestTransformFielFormatMap(t *testing.T) { + tests := []struct { + commonField common.Field + expected common.MapStr + }{ + { + commonField: common.Field{Name: "c"}, + expected: common.MapStr{}, + }, + { + commonField: common.Field{Name: "c", Format: "url"}, + expected: common.MapStr{"c": common.MapStr{"id": "url"}}, + }, + { + commonField: common.Field{ + Name: "c", + Pattern: "p", + }, + expected: common.MapStr{ + "c": common.MapStr{ + "params": common.MapStr{"pattern": "p"}, + }, + }, + }, + { + commonField: common.Field{ + Name: "c", + Format: "url", + Pattern: "p", + }, + expected: common.MapStr{ + "c": common.MapStr{ + "id": "url", + "params": common.MapStr{"pattern": "p"}, + }, + }, + }, + { + commonField: common.Field{ + Name: "c", + Format: "url", + InputFormat: "string", + }, + expected: common.MapStr{ + "c": common.MapStr{ + "id": "url", + "params": common.MapStr{ + "inputFormat": "string", + }, + }, + }, + }, + { + commonField: common.Field{ + Name: "c", + Format: "url", + Pattern: "[^-]", + InputFormat: "string", + }, + expected: common.MapStr{ + "c": common.MapStr{ + "id": "url", + "params": common.MapStr{ + "pattern": "[^-]", + }, + }, + }, + }, + { + commonField: common.Field{ + Name: "c", + InputFormat: "string", + }, + expected: common.MapStr{}, + }, + } + for idx, test := range tests { + out := TransformFields("", "", common.Fields{test.commonField})["fieldFormatMap"] + assert.Equal(t, test.expected, out, fmt.Sprintf("Failed for idx %v", idx)) + } +} diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 1591d5f4f4c..f550b23863c 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -293,7 +293,7 @@ endif @# Generate Kibana index pattern @mkdir -p $(PWD)/_meta/kibana/5.x/index-pattern @mkdir -p $(PWD)/_meta/kibana/default/index-pattern - @${PYTHON_ENV}/bin/python ${ES_BEATS}/libbeat/scripts/generate_index_pattern.py --index '${BEAT_INDEX_PREFIX}-*' --beat-name ${BEAT_NAME} --beat $(PWD) --version ${BEAT_VERSION} + go run -v ${ES_BEATS}/libbeat/scripts/generate_index_pattern.go '${BEAT_INDEX_PREFIX}-*' ${BEAT_NAME} $(PWD) ${BEAT_VERSION} # Runs the post-update target if it exists. Will not return errors @-$(MAKE) post-update diff --git a/libbeat/scripts/generate_index_pattern.go b/libbeat/scripts/generate_index_pattern.go new file mode 100644 index 00000000000..66ae0cc0533 --- /dev/null +++ b/libbeat/scripts/generate_index_pattern.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "os" + + "github.com/elastic/beats/libbeat/kibana" +) + +func main() { + index := kibana.Index{ + IndexName: os.Args[1], + BeatDir: os.Args[3], + BeatName: os.Args[2], + Version: os.Args[4], + } + indices, err := index.Create() + if err != nil { + panic(err) + } + for _, i := range indices { + fmt.Println("-- The index pattern was created under ", i) + } +} From b98ba2ddd30d8f6c9dede78528ba7eb8b8e389b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 2 Oct 2017 16:14:53 +0200 Subject: [PATCH 60/96] Fix minor typos (#5286) --- libbeat/cmd/instance/beat.go | 2 +- libbeat/cmd/run.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 88fe86bff32..731f9330109 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -370,7 +370,7 @@ func (b *Beat) handleFlags() error { flag.Parse() if printVersion { - cfgwarn.Deprecate("6.0", "-version flag has been deprectad, use version subcommand") + cfgwarn.Deprecate("6.0", "-version flag has been deprecated, use version subcommand") fmt.Printf("%s version %s (%s), libbeat %s\n", b.Info.Beat, b.Info.Version, runtime.GOARCH, version.GetDefaultVersion()) return beat.GracefulExit diff --git a/libbeat/cmd/run.go b/libbeat/cmd/run.go index 9abe23f535a..6ec04126740 100644 --- a/libbeat/cmd/run.go +++ b/libbeat/cmd/run.go @@ -34,8 +34,8 @@ func genRunCmd(name, idxPrefix, version string, beatCreator beat.Creator, runFla runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("configtest")) runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("version")) - runCmd.Flags().MarkDeprecated("version", "version flag has been deprectad, use version subcommand") - runCmd.Flags().MarkDeprecated("configtest", "configtest flag has been deprectad, use test config subcommand") + runCmd.Flags().MarkDeprecated("version", "version flag has been deprecated, use version subcommand") + runCmd.Flags().MarkDeprecated("configtest", "configtest flag has been deprecated, use test config subcommand") if runFlags != nil { runCmd.Flags().AddFlagSet(runFlags) From 3c075f76d4540348f3506f5d7afc80456e0755be Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 2 Oct 2017 17:23:19 +0200 Subject: [PATCH 61/96] Add filesystem stats to ES node_stats metricset (#5230) Closes https://github.com/elastic/beats/issues/5209 --- metricbeat/docs/fields.asciidoc | 34 ++++ .../elasticsearch/node_stats/_meta/data.json | 145 ++++++++++-------- .../elasticsearch/node_stats/_meta/fields.yml | 18 +++ .../module/elasticsearch/node_stats/data.go | 13 ++ 4 files changed, 144 insertions(+), 66 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5088d17e4f5..6afabb028f4 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2743,6 +2743,40 @@ type: long +[float] +== fs.summary fields + +File system summary + + + +[float] +=== `elasticsearch.node.stats.fs.summary.total.bytes` + +type: long + +format: bytes + + + +[float] +=== `elasticsearch.node.stats.fs.summary.free.bytes` + +type: long + +format: bytes + + + +[float] +=== `elasticsearch.node.stats.fs.summary.available.bytes` + +type: long + +format: bytes + + + [[exported-fields-golang]] == Golang fields diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data.json b/metricbeat/module/elasticsearch/node_stats/_meta/data.json index a53b7a5376b..870db723a6d 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/data.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/data.json @@ -1,105 +1,118 @@ { "@timestamp": "2016-05-23T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "elasticsearch": { - "cluster": { - "name": "elasticsearch" - }, "node": { - "name": "8zgOwJ24TMamDDH9amWINQ", "stats": { - "indices": { - "docs": { - "count": 0, - "deleted": 0 - }, - "segments": { - "count": 0, - "memory": { - "bytes": 0 - } - }, - "store": { - "size": { - "bytes": 0 - } - } - }, "jvm": { - "gc": { - "collectors": { - "old": { - "collection": { - "count": 1, - "ms": 80 - } - }, - "young": { - "collection": { - "count": 4, - "ms": 151 - } - } - } - }, "mem": { "pools": { - "old": { + "young": { + "used": { + "bytes": 125648960 + }, "max": { - "bytes": 362414080 + "bytes": 139591680 }, "peak": { - "bytes": 20109096 + "bytes": 139591680 }, "peak_max": { - "bytes": 362414080 - }, - "used": { - "bytes": 20109096 + "bytes": 139591680 } }, "survivor": { - "max": { + "peak_max": { "bytes": 17432576 }, - "peak": { - "bytes": 17432576 + "used": { + "bytes": 12279056 }, - "peak_max": { + "max": { "bytes": 17432576 }, - "used": { - "bytes": 17432560 + "peak": { + "bytes": 17432568 } }, - "young": { + "old": { + "used": { + "bytes": 176528696 + }, "max": { - "bytes": 139591680 + "bytes": 362414080 }, "peak": { - "bytes": 139591680 + "bytes": 176528696 }, "peak_max": { - "bytes": 139591680 - }, - "used": { - "bytes": 98778408 + "bytes": 362414080 + } + } + } + }, + "gc": { + "collectors": { + "old": { + "collection": { + "count": 3, + "ms": 693 + } + }, + "young": { + "collection": { + "ms": 10691, + "count": 209 } } } } + }, + "indices": { + "docs": { + "deleted": 42, + "count": 3093 + }, + "store": { + "size": { + "bytes": 2062479 + } + }, + "segments": { + "count": 17, + "memory": { + "bytes": 179535 + } + } + }, + "fs": { + "summary": { + "total": { + "bytes": 67371577344 + }, + "free": { + "bytes": 56422297600 + }, + "available": { + "bytes": 52969598976 + } + } } - } + }, + "name": "2jwM3tNWSr6tFlL8fLKtSg" + }, + "cluster": { + "name": "elasticsearch" } }, "metricset": { - "host": "127.0.0.1:9200", + "rtt": 115, + "namespace": "node.stats", "module": "elasticsearch", "name": "node_stats", - "namespace": "node.stats", - "rtt": 115 + "host": "127.0.0.1:9200" + }, + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" } -} \ No newline at end of file +} diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml b/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml index 5506bec2109..967034d2afa 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml @@ -137,3 +137,21 @@ - name: ms type: long description: > + + - name: fs.summary + type: group + description: > + File system summary + fields: + - name: total.bytes + type: long + format: bytes + description: > + - name: free.bytes + type: long + format: bytes + description: > + - name: available.bytes + type: long + format: bytes + description: > diff --git a/metricbeat/module/elasticsearch/node_stats/data.go b/metricbeat/module/elasticsearch/node_stats/data.go index 05d1f2c16c5..5c316b39824 100644 --- a/metricbeat/module/elasticsearch/node_stats/data.go +++ b/metricbeat/module/elasticsearch/node_stats/data.go @@ -43,6 +43,19 @@ var ( }, }), }), + "fs": c.Dict("fs", s.Schema{ + "summary": c.Dict("total", s.Schema{ + "total": s.Object{ + "bytes": c.Int("total_in_bytes"), + }, + "free": s.Object{ + "bytes": c.Int("free_in_bytes"), + }, + "available": s.Object{ + "bytes": c.Int("available_in_bytes"), + }, + }), + }), } poolSchema = s.Schema{ From ffca490b8c5a3142191d7cc4de7b72ea5f59d6c5 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Mon, 2 Oct 2017 21:36:41 +0200 Subject: [PATCH 62/96] Align kibana index generation to beats standard. (#5290) * Move generator script to dev-tools. * Add NewGenerator method which returns struct instead of init method. --- .../kibana_index_pattern.go | 49 +++++ .../{index.go => index_pattern_generator.go} | 72 +++--- ...est.go => index_pattern_generator_test.go} | 82 ++++--- libbeat/kibana/transform.go | 58 +++-- libbeat/kibana/transform_test.go | 20 +- libbeat/scripts/Makefile | 2 +- libbeat/scripts/generate_index_pattern.go | 24 -- libbeat/scripts/generate_index_pattern.py | 205 ------------------ 8 files changed, 184 insertions(+), 328 deletions(-) create mode 100644 dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go rename libbeat/kibana/{index.go => index_pattern_generator.go} (58%) rename libbeat/kibana/{index_test.go => index_pattern_generator_test.go} (67%) delete mode 100644 libbeat/scripts/generate_index_pattern.go delete mode 100644 libbeat/scripts/generate_index_pattern.py diff --git a/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go b/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go new file mode 100644 index 00000000000..f90b93faf43 --- /dev/null +++ b/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go @@ -0,0 +1,49 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/elastic/beats/libbeat/kibana" + "github.com/elastic/beats/libbeat/version" +) + +func main() { + beatVersion := version.GetDefaultVersion() + index := flag.String("index", "", "The name of the index pattern. (required)") + beatName := flag.String("beat-name", "", "The name of the beat. (required)") + beatDir := flag.String("beat-dir", "", "The local beat directory. (required)") + version := flag.String("version", beatVersion, "The beat version.") + flag.Parse() + + if *index == "" { + fmt.Fprint(os.Stderr, "The name of the index pattern msut be set.") + os.Exit(1) + } + + if *beatName == "" { + fmt.Fprint(os.Stderr, "The name of the beat must be set.") + os.Exit(1) + } + + if *beatDir == "" { + fmt.Fprint(os.Stderr, "The beat directory must be set.") + os.Exit(1) + } + + indexPatternGenerator, err := kibana.NewGenerator(*index, *beatName, *beatDir, *version) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + pattern, err := indexPatternGenerator.Generate() + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + for _, p := range pattern { + fmt.Fprintf(os.Stdout, "-- The index pattern was created under %v\n", p) + } +} diff --git a/libbeat/kibana/index.go b/libbeat/kibana/index_pattern_generator.go similarity index 58% rename from libbeat/kibana/index.go rename to libbeat/kibana/index_pattern_generator.go index 68d033e2ef5..7f756709b2a 100644 --- a/libbeat/kibana/index.go +++ b/libbeat/kibana/index_pattern_generator.go @@ -2,7 +2,6 @@ package kibana import ( "encoding/json" - "errors" "io/ioutil" "os" "path/filepath" @@ -11,43 +10,56 @@ import ( "github.com/elastic/beats/libbeat/common" ) -type Index struct { - Version string - IndexName string - BeatDir string - BeatName string - +type IndexPatternGenerator struct { + indexName string + version string fieldsYaml string targetDirDefault string targetDir5x string targetFilename string } -// Create the Index-Pattern for Kibana for 5.x and default. -func (i *Index) Create() ([]string, error) { - indices := []string{} +// Create an instance of the Kibana Index Pattern Generator +func NewGenerator(indexName, beatName, beatDir, version string) (*IndexPatternGenerator, error) { + beatName = clean(beatName) - err := i.init() - if err != nil { + fieldsYaml := filepath.Join(beatDir, "fields.yml") + if _, err := os.Stat(fieldsYaml); err != nil { return nil, err } + return &IndexPatternGenerator{ + indexName: indexName, + version: version, + fieldsYaml: fieldsYaml, + targetDirDefault: createTargetDir(beatDir, "default"), + targetDir5x: createTargetDir(beatDir, "5.x"), + targetFilename: beatName + ".json", + }, nil +} + +// Create the Index-Pattern for Kibana for 5.x and default. +func (i *IndexPatternGenerator) Generate() ([]string, error) { + var indices []string + commonFields, err := common.LoadFieldsYaml(i.fieldsYaml) if err != nil { return nil, err } - transformed := TransformFields("@timestamp", i.IndexName, commonFields) + transformer := NewTransformer("@timestamp", i.indexName, commonFields) + transformed := transformer.TransformFields() - if fieldsBytes, err := json.Marshal(transformed["fields"]); err != nil { + fieldsBytes, err := json.Marshal(transformed["fields"]) + if err != nil { return nil, err - } else { - transformed["fields"] = string(fieldsBytes) } - if fieldFormatBytes, err := json.Marshal(transformed["fieldFormatMap"]); err != nil { + transformed["fields"] = string(fieldsBytes) + + fieldFormatBytes, err := json.Marshal(transformed["fieldFormatMap"]) + if err != nil { return nil, err - } else { - transformed["fieldFormatMap"] = string(fieldFormatBytes) } + transformed["fieldFormatMap"] = string(fieldFormatBytes) file5x := filepath.Join(i.targetDir5x, i.targetFilename) err = dumpToFile(file5x, transformed) @@ -57,11 +69,11 @@ func (i *Index) Create() ([]string, error) { indices = append(indices, file5x) out := common.MapStr{ - "version": i.Version, + "version": i.version, "objects": []common.MapStr{ common.MapStr{ "type": "index-pattern", - "id": i.IndexName, + "id": i.indexName, "version": 1, "attributes": transformed, }, @@ -77,24 +89,6 @@ func (i *Index) Create() ([]string, error) { return indices, nil } -func (i *Index) init() error { - if i.Version == "" || i.IndexName == "" || i.BeatDir == "" || i.BeatName == "" { - return errors.New("RequiredParams: Version, IndexName, BeatDir and BeatName") - } - i.BeatName = clean(i.BeatName) - - i.fieldsYaml = filepath.Join(i.BeatDir, "fields.yml") - if _, err := os.Stat(i.fieldsYaml); err != nil { - return err - } - - i.targetDirDefault = createTargetDir(i.BeatDir, "default") - i.targetDir5x = createTargetDir(i.BeatDir, "5.x") - i.targetFilename = i.BeatName + ".json" - - return nil -} - func clean(name string) string { reg := regexp.MustCompile("[^a-zA-Z0-9_]+") return reg.ReplaceAllString(name, "") diff --git a/libbeat/kibana/index_test.go b/libbeat/kibana/index_pattern_generator_test.go similarity index 67% rename from libbeat/kibana/index_test.go rename to libbeat/kibana/index_pattern_generator_test.go index faa5b94ba6d..54d26938c52 100644 --- a/libbeat/kibana/index_test.go +++ b/libbeat/kibana/index_pattern_generator_test.go @@ -14,49 +14,32 @@ import ( "github.com/elastic/beats/libbeat/common" ) -func TestAllArgsSet(t *testing.T) { +func TestNewGenerator(t *testing.T) { beatDir := tmpPath() defer teardown(beatDir) - tests := []struct { - Index Index - }{ - {Index: Index{IndexName: "beat-index", BeatDir: beatDir, BeatName: "mybeat."}}, - {Index: Index{Version: "6.0", BeatDir: beatDir, BeatName: "mybeat."}}, - {Index: Index{Version: "6.0", IndexName: "beat-index", BeatName: "mybeat."}}, - {Index: Index{Version: "6.0", IndexName: "beat-index", BeatDir: beatDir}}, - } - for idx, test := range tests { - err := test.Index.init() - msg := fmt.Sprintf("(%v): Should have raised error", idx) - assert.Error(t, err, msg) - } -} - -func TestInit(t *testing.T) { - beatDir := tmpPath() - defer teardown(beatDir) // checks for fields.yml - idx := Index{Version: "7.0", IndexName: "beat-index", BeatDir: filepath.Join(beatDir, "notexistent"), BeatName: "mybeat."} - err := idx.init() + generator, err := NewGenerator("beat-index", "mybeat.", filepath.Join(beatDir, "notexistent"), "7.0") assert.Error(t, err) - idx = Index{Version: "7.0", IndexName: "beat-index", BeatDir: beatDir, BeatName: "mybeat."} - err = idx.init() + generator, err = NewGenerator("beat-index", "mybeat.", beatDir, "7.0") assert.NoError(t, err) + assert.Equal(t, "7.0", generator.version) + assert.Equal(t, "beat-index", generator.indexName) + assert.Equal(t, filepath.Join(beatDir, "fields.yml"), generator.fieldsYaml) // creates file dir and sets name expectedDir := filepath.Join(beatDir, "_meta/kibana/default/index-pattern") - assert.Equal(t, expectedDir, idx.targetDirDefault) - _, err = os.Stat(idx.targetDirDefault) + assert.Equal(t, expectedDir, generator.targetDirDefault) + _, err = os.Stat(generator.targetDirDefault) assert.NoError(t, err) expectedDir = filepath.Join(beatDir, "_meta/kibana/5.x/index-pattern") - assert.Equal(t, expectedDir, idx.targetDir5x) - _, err = os.Stat(idx.targetDir5x) + assert.Equal(t, expectedDir, generator.targetDir5x) + _, err = os.Stat(generator.targetDir5x) assert.NoError(t, err) - assert.Equal(t, "mybeat.json", idx.targetFilename) + assert.Equal(t, "mybeat.json", generator.targetFilename) } func TestCleanName(t *testing.T) { @@ -75,12 +58,49 @@ func TestCleanName(t *testing.T) { } } -func TestDefault(t *testing.T) { +func TestGenerateFieldsYaml(t *testing.T) { + beatDir := tmpPath() + defer teardown(beatDir) + generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1") + _, err = generator.Generate() + assert.NoError(t, err) + + generator.fieldsYaml = "" + _, err = generator.Generate() + assert.Error(t, err) +} + +func TestDumpToFile5x(t *testing.T) { beatDir := tmpPath() defer teardown(beatDir) + generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1") + _, err = generator.Generate() + assert.NoError(t, err) + + generator.targetDir5x = "./non-existing/something" + _, err = generator.Generate() + assert.Error(t, err) +} - index := Index{Version: "7.0.0-alpha1", IndexName: "metricbeat-*", BeatDir: beatDir, BeatName: "metric beat !"} - index.Create() +func TestDumpToFileDefault(t *testing.T) { + beatDir := tmpPath() + defer teardown(beatDir) + generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1") + _, err = generator.Generate() + assert.NoError(t, err) + + generator.targetDirDefault = "./non-existing/something" + _, err = generator.Generate() + assert.Error(t, err) +} + +func TestGenerate(t *testing.T) { + beatDir := tmpPath() + defer teardown(beatDir) + generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1") + pattern, err := generator.Generate() + assert.NoError(t, err) + assert.Equal(t, 2, len(pattern)) tests := []map[string]string{ {"existing": "metricbeat-5x-old.json", "created": "_meta/kibana/5.x/index-pattern/metricbeat.json"}, diff --git a/libbeat/kibana/transform.go b/libbeat/kibana/transform.go index 46938294cd1..68adb50af58 100644 --- a/libbeat/kibana/transform.go +++ b/libbeat/kibana/transform.go @@ -7,46 +7,62 @@ import ( "github.com/elastic/beats/libbeat/common" ) -func TransformFields(timeFieldName string, title string, commonFields common.Fields) common.MapStr { - fields := []common.MapStr{} - fieldFormatMap := common.MapStr{} - keys := common.MapStr{} +type Transformer struct { + fields common.Fields + transformedFields []common.MapStr + transformedFieldFormatMap common.MapStr + timeFieldName string + title string + keys common.MapStr +} + +func NewTransformer(timeFieldName, title string, fields common.Fields) *Transformer { + return &Transformer{ + fields: fields, + timeFieldName: timeFieldName, + title: title, + transformedFields: []common.MapStr{}, + transformedFieldFormatMap: common.MapStr{}, + keys: common.MapStr{}, + } +} - transformFields(keys, commonFields, &fields, fieldFormatMap, "") +func (t *Transformer) TransformFields() common.MapStr { + t.transformFields(t.fields, "") // add some meta fields truthy := true falsy := false - add(common.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap) - add(common.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}, &fields, fieldFormatMap) - add(common.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap) - add(common.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap) + t.add(common.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(common.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}) + t.add(common.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(common.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) return common.MapStr{ - "timeFieldName": timeFieldName, - "title": title, - "fields": fields, - "fieldFormatMap": fieldFormatMap, + "timeFieldName": t.timeFieldName, + "title": t.title, + "fields": t.transformedFields, + "fieldFormatMap": t.transformedFieldFormatMap, } } -func transformFields(keys common.MapStr, commonFields common.Fields, fields *[]common.MapStr, fieldFormatMap common.MapStr, path string) { +func (t *Transformer) transformFields(commonFields common.Fields, path string) { for _, f := range commonFields { f.Path = f.Name if path != "" { f.Path = path + "." + f.Name } - if keys[f.Path] != nil { + if t.keys[f.Path] != nil { msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.", f.Path) panic(errors.New(msg)) } if f.Type == "group" { - transformFields(keys, f.Fields, fields, fieldFormatMap, f.Path) + t.transformFields(f.Fields, f.Path) } else { // set default values (as done in python script) - keys[f.Path] = true + t.keys[f.Path] = true truthy := true falsy := false @@ -55,16 +71,16 @@ func transformFields(keys common.MapStr, commonFields common.Fields, fields *[]c f.DocValues = &truthy f.Searchable = &truthy f.Aggregatable = &truthy - add(f, fields, fieldFormatMap) + t.add(f) } } } -func add(f common.Field, fields *[]common.MapStr, fieldFormatMap common.MapStr) { +func (t *Transformer) add(f common.Field) { field, fieldFormat := transformField(f) - *fields = append(*fields, field) + t.transformedFields = append(t.transformedFields, field) if fieldFormat != nil { - fieldFormatMap[field["name"].(string)] = fieldFormat + t.transformedFieldFormatMap[field["name"].(string)] = fieldFormat } } diff --git a/libbeat/kibana/transform_test.go b/libbeat/kibana/transform_test.go index 0f5f58b0ec6..7718a0aaa78 100644 --- a/libbeat/kibana/transform_test.go +++ b/libbeat/kibana/transform_test.go @@ -16,7 +16,8 @@ var ( ) func TestEmpty(t *testing.T) { - out := TransformFields("name", "title", common.Fields{}) + trans := NewTransformer("name", "title", common.Fields{}) + out := trans.TransformFields() expected := common.MapStr{ "timeFieldName": "name", "title": "title", @@ -76,7 +77,8 @@ func TestErrors(t *testing.T) { common.Field{Name: "context", Path: "something"}, common.Field{Name: "context", Path: "something", Type: "keyword"}, } - assert.Panics(t, func() { TransformFields("", "", commonFields) }) + trans := NewTransformer("name", "title", commonFields) + assert.Panics(t, func() { trans.TransformFields() }) } func TestTransformTypes(t *testing.T) { @@ -100,7 +102,8 @@ func TestTransformTypes(t *testing.T) { {commonField: common.Field{Type: "invalid"}, expected: nil}, } for idx, test := range tests { - out := TransformFields("", "", common.Fields{test.commonField})["fields"].([]common.MapStr)[0] + trans := NewTransformer("name", "title", common.Fields{test.commonField}) + out := trans.TransformFields()["fields"].([]common.MapStr)[0] assert.Equal(t, test.expected, out["type"], fmt.Sprintf("Failed for idx %v", idx)) } } @@ -142,7 +145,8 @@ func TestTransformGroup(t *testing.T) { }, } for idx, test := range tests { - out := TransformFields("", "", test.commonFields)["fields"].([]common.MapStr) + trans := NewTransformer("name", "title", test.commonFields) + out := trans.TransformFields()["fields"].([]common.MapStr) assert.Equal(t, len(test.expected)+ctMetaData, len(out)) for i, e := range test.expected { assert.Equal(t, e, out[i]["name"], fmt.Sprintf("Failed for idx %v", idx)) @@ -190,13 +194,14 @@ func TestTransformMisc(t *testing.T) { {commonField: common.Field{}, expected: false, attr: "scripted"}, } for idx, test := range tests { - out := TransformFields("", "", common.Fields{test.commonField})["fields"].([]common.MapStr)[0] + trans := NewTransformer("", "", common.Fields{test.commonField}) + out := trans.TransformFields()["fields"].([]common.MapStr)[0] msg := fmt.Sprintf("(%v): expected '%s' to be <%v> but was <%v>", idx, test.attr, test.expected, out[test.attr]) assert.Equal(t, test.expected, out[test.attr], msg) } } -func TestTransformFielFormatMap(t *testing.T) { +func TestTransformFieldFormatMap(t *testing.T) { tests := []struct { commonField common.Field expected common.MapStr @@ -273,7 +278,8 @@ func TestTransformFielFormatMap(t *testing.T) { }, } for idx, test := range tests { - out := TransformFields("", "", common.Fields{test.commonField})["fieldFormatMap"] + trans := NewTransformer("", "", common.Fields{test.commonField}) + out := trans.TransformFields()["fieldFormatMap"] assert.Equal(t, test.expected, out, fmt.Sprintf("Failed for idx %v", idx)) } } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index f550b23863c..0677719b5b3 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -293,7 +293,7 @@ endif @# Generate Kibana index pattern @mkdir -p $(PWD)/_meta/kibana/5.x/index-pattern @mkdir -p $(PWD)/_meta/kibana/default/index-pattern - go run -v ${ES_BEATS}/libbeat/scripts/generate_index_pattern.go '${BEAT_INDEX_PREFIX}-*' ${BEAT_NAME} $(PWD) ${BEAT_VERSION} + @go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go -index '${BEAT_INDEX_PREFIX}-*' -beat-name ${BEAT_NAME} -beat-dir $(PWD) -version ${BEAT_VERSION} # Runs the post-update target if it exists. Will not return errors @-$(MAKE) post-update diff --git a/libbeat/scripts/generate_index_pattern.go b/libbeat/scripts/generate_index_pattern.go deleted file mode 100644 index 66ae0cc0533..00000000000 --- a/libbeat/scripts/generate_index_pattern.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/elastic/beats/libbeat/kibana" -) - -func main() { - index := kibana.Index{ - IndexName: os.Args[1], - BeatDir: os.Args[3], - BeatName: os.Args[2], - Version: os.Args[4], - } - indices, err := index.Create() - if err != nil { - panic(err) - } - for _, i := range indices { - fmt.Println("-- The index pattern was created under ", i) - } -} diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py deleted file mode 100644 index 890ba07af17..00000000000 --- a/libbeat/scripts/generate_index_pattern.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python - -""" -This script generates the index-pattern for Kibana from -the fields.yml file. -""" - -import yaml -import argparse -import string -import re -import json -import os -import errno -import sys - -unique_fields = [] - - -def fields_to_json(fields, section, path, output): - - # Need in case there are no fields - if section["fields"] is None: - section["fields"] = {} - - for field in section["fields"]: - if path == "": - newpath = field["name"] - else: - newpath = path + "." + field["name"] - - if "type" in field and field["type"] == "group": - fields_to_json(fields, field, newpath, output) - else: - field_to_json(fields, field, newpath, output) - - -def field_to_json(fields, desc, path, output, - indexed=True, analyzed=False, doc_values=True, - searchable=True, aggregatable=True): - - if path in unique_fields: - print("ERROR: Field {} is duplicated. Please delete it and try again. Fields already are {}".format( - path, ", ".join(fields))) - sys.exit(1) - else: - fields.append(path) - - field = { - "name": path, - "count": 0, - "scripted": False, - "indexed": indexed, - "analyzed": analyzed, - "doc_values": doc_values, - "searchable": searchable, - "aggregatable": aggregatable, - } - # find the kibana types based on the field type - if "type" in desc: - if desc["type"] in ["half_float", "scaled_float", "float", "integer", "long", "short", "byte"]: - field["type"] = "number" - elif desc["type"] in ["text", "keyword"]: - field["type"] = "string" - if desc["type"] == "text": - field["aggregatable"] = False - elif desc["type"] == "date": - field["type"] = "date" - elif desc["type"] == "geo_point": - field["type"] = "geo_point" - else: - field["type"] = "string" - - output["fields"].append(field) - - if "format" in desc or "pattern" in desc: - fieldFormat = {} - - if "format" in desc: - fieldFormat["id"] = desc["format"] - - if "input_format" in desc: - fieldFormat["params"] = { - "inputFormat": desc["input_format"] - } - - if "pattern" in desc: - fieldFormat["params"] = { - "pattern": desc["pattern"] - } - - output["fieldFormatMap"][path] = fieldFormat - - -def fields_to_index_pattern(version, args, input): - - docs = yaml.load(input) - fields = [] - - if docs is None: - print("fields.yml is empty. Cannot generate index-pattern") - return - - attributes = { - "fields": [], - "fieldFormatMap": {}, - "timeFieldName": "@timestamp", - "title": args.index, - - } - - for k, section in enumerate(docs): - fields_to_json(fields, section, "", attributes) - - # add meta fields - - field_to_json(fields, {"name": "_id", "type": "keyword"}, "_id", - attributes, - indexed=False, analyzed=False, doc_values=False, - searchable=False, aggregatable=False) - - field_to_json(fields, {"name": "_type", "type": "keyword"}, "_type", - attributes, - indexed=False, analyzed=False, doc_values=False, - searchable=True, aggregatable=True) - - field_to_json(fields, {"name": "_index", "type": "keyword"}, "_index", - attributes, - indexed=False, analyzed=False, doc_values=False, - searchable=False, aggregatable=False) - - field_to_json(fields, {"name": "_score", "type": "integer"}, "_score", - attributes, - indexed=False, analyzed=False, doc_values=False, - searchable=False, aggregatable=False) - - attributes["fields"] = json.dumps(attributes["fields"]) - attributes["fieldFormatMap"] = json.dumps(attributes["fieldFormatMap"]) - - if version == "5.x": - return attributes - - return { - "version": args.version, - "objects": [{ - "type": "index-pattern", - "id": args.index, - "version": 1, - "attributes": attributes, - }] - - - } - - -def get_index_pattern_name(index): - - allow = string.ascii_letters + string.digits + "_" - return re.sub('[^%s]' % allow, '', index) - - -def dump_index_pattern(args, version, output): - - fileName = get_index_pattern_name(args.beat_name) - target_dir = os.path.join(args.beat, "_meta", "kibana", version, "index-pattern") - target_file = os.path.join(target_dir, fileName + ".json") - - try: - os.makedirs(target_dir) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - - output = json.dumps(output, indent=2) - - with open(target_file, 'w') as f: - f.write(output) - - print("The index pattern was created under {}".format(target_file)) - return target_file - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser( - description="Generates the index-pattern for a Beat.") - parser.add_argument("--version", help="Beat version") - parser.add_argument("--index", help="The name of the index-pattern") - parser.add_argument("--beat", help="Local Beat directory") - parser.add_argument("--beat-name", help="Name of the beat") - - args = parser.parse_args() - - fields_yml = args.beat + "/fields.yml" - - # generate the index-pattern content - with open(fields_yml, 'r') as f: - fields = f.read() - - # with open(target, 'w') as output: - output = fields_to_index_pattern("default", args, fields) - dump_index_pattern(args, "default", output) - - output5x = fields_to_index_pattern("5.x", args, fields) - dump_index_pattern(args, "5.x", output5x) From 8d79c8f364bfd085c303fa295496ffd0d044ff3e Mon Sep 17 00:00:00 2001 From: Gilad Peleg Date: Tue, 3 Oct 2017 09:51:04 +0300 Subject: [PATCH 63/96] Add additional php-fpm pool status kpis for Metricbeat module (#5287) --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 66 +++++++++++++++++++ .../module/php_fpm/pool/_meta/data.json | 22 +++++-- .../module/php_fpm/pool/_meta/fields.yml | 36 ++++++++++ metricbeat/module/php_fpm/pool/data.go | 20 ++++-- 5 files changed, 132 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 20bcd934d79..097e56781f7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -105,6 +105,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add system uptime metricset. {issue}[4848[4848] - Add `filesystem.ignore_types` to system module for ignoring filesystem types. {issue}4685[4685] - Add experimental `queue` metricset to RabbitMQ module. {pull}4788[4788] +- Add additional php-fpm pool status kpis for Metricbeat module {pull}5287[5287] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 6afabb028f4..2301923d53b 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -7042,6 +7042,14 @@ type: keyword The name of the pool. +[float] +=== `php_fpm.pool.process_manager` + +type: keyword + +Static, dynamic or ondemand. + + [float] == connections fields @@ -7065,6 +7073,22 @@ type: long The current number of connections that have been initiated, but not yet accepted. If this value is non-zero it typically means that all the available server processes are currently busy, and there are no processes available to serve the next request. Raising `pm.max_children` (provided the server can handle it) should help keep this number low. This property follows from the fact that PHP-FPM listens via a socket (TCP or file based), and thus inherits some of the characteristics of sockets. +[float] +=== `php_fpm.pool.connections.max_listen_queue` + +type: long + +The maximum number of requests in the queue of pending connections since FPM has started. + + +[float] +=== `php_fpm.pool.connections.listen_queue_len` + +type: long + +The size of the socket queue of pending connections. + + [float] == processes fields @@ -7088,6 +7112,30 @@ type: long The number of servers current processing a page - the minimum is `1` (so even on a fully idle server, the result will be not read `0`). +[float] +=== `php_fpm.pool.processes.total` + +type: long + +The number of idle + active processes. + + +[float] +=== `php_fpm.pool.processes.max_active` + +type: long + +The maximum number of active processes since FPM has started. + + +[float] +=== `php_fpm.pool.processes.max_children_reached` + +type: long + +Number of times, the process limit has been reached, when pm tries to start more children (works only for pm 'dynamic' and 'ondemand'). + + [float] === `php_fpm.pool.slow_requests` @@ -7096,6 +7144,24 @@ type: long The number of times a request execution time has exceeded `request_slowlog_timeout`. +[float] +=== `php_fpm.pool.start_since` + +type: long + +Number of seconds since FPM has started. + + +[float] +=== `php_fpm.pool.start_time` + +type: date + +format: epoch_second + +The date and time FPM has started. + + [[exported-fields-postgresql]] == PostgreSQL fields diff --git a/metricbeat/module/php_fpm/pool/_meta/data.json b/metricbeat/module/php_fpm/pool/_meta/data.json index 8616a772903..0fe1c239c8d 100644 --- a/metricbeat/module/php_fpm/pool/_meta/data.json +++ b/metricbeat/module/php_fpm/pool/_meta/data.json @@ -1,5 +1,5 @@ { - "@timestamp": "2016-05-23T08:05:34.853Z", + "@timestamp": "2017-09-30T12:03:14.242Z", "beat": { "hostname": "host.example.com", "name": "host.example.com" @@ -13,16 +13,24 @@ "php_fpm": { "pool": { "connections": { - "accepted": 13, + "accepted": 28729, + "listen_queue_len": 128, + "max_listen_queue": 1, "queued": 0 }, - "pool": "www", + "name": "www", + "process_manager": "dynamic", "processes": { - "active": 1, - "idle": 2 + "active": 2, + "idle": 3, + "max_active": 8, + "max_children_reached": 2, + "total": 5 }, - "slow_requests": 0 + "slow_requests": 0, + "start_since": 171707, + "start_time": 1506604034 } }, "type": "metricsets" -} \ No newline at end of file +} diff --git a/metricbeat/module/php_fpm/pool/_meta/fields.yml b/metricbeat/module/php_fpm/pool/_meta/fields.yml index c4e2a096b0b..a4a43f884a5 100644 --- a/metricbeat/module/php_fpm/pool/_meta/fields.yml +++ b/metricbeat/module/php_fpm/pool/_meta/fields.yml @@ -8,6 +8,10 @@ type: keyword description: > The name of the pool. + - name: process_manager + type: keyword + description: > + Static, dynamic or ondemand. - name: connections type: group description: > @@ -29,6 +33,15 @@ keep this number low. This property follows from the fact that PHP-FPM listens via a socket (TCP or file based), and thus inherits some of the characteristics of sockets. + - name: max_listen_queue + type: long + description: > + The maximum number of requests in the queue of pending + connections since FPM has started. + - name: listen_queue_len + type: long + description: > + The size of the socket queue of pending connections. - name: processes type: group description: > @@ -46,8 +59,31 @@ description: > The number of servers current processing a page - the minimum is `1` (so even on a fully idle server, the result will be not read `0`). + - name: total + type: long + description: > + The number of idle + active processes. + - name: max_active + type: long + description: > + The maximum number of active processes since FPM has started. + - name: max_children_reached + type: long + description: > + Number of times, the process limit has been reached, + when pm tries to start more children (works only for + pm 'dynamic' and 'ondemand'). - name: slow_requests type: long description: > The number of times a request execution time has exceeded `request_slowlog_timeout`. + - name: start_since + type: long + description: > + Number of seconds since FPM has started. + - name: start_time + type: date + format: epoch_second + description: > + The date and time FPM has started. diff --git a/metricbeat/module/php_fpm/pool/data.go b/metricbeat/module/php_fpm/pool/data.go index 74f66865aa1..a44eb311279 100644 --- a/metricbeat/module/php_fpm/pool/data.go +++ b/metricbeat/module/php_fpm/pool/data.go @@ -7,15 +7,23 @@ import ( var ( schema = s.Schema{ - "name": c.Str("pool"), + "name": c.Str("pool"), + "process_manager": c.Str("process manager"), + "slow_requests": c.Int("slow requests"), + "start_time": c.Int("start time"), + "start_since": c.Int("start since"), "connections": s.Object{ - "accepted": c.Int("accepted conn"), - "queued": c.Int("listen queue"), + "accepted": c.Int("accepted conn"), + "listen_queue_len": c.Int("listen queue len"), + "max_listen_queue": c.Int("max listen queue"), + "queued": c.Int("listen queue"), }, "processes": s.Object{ - "idle": c.Int("idle processes"), - "active": c.Int("active processes"), + "active": c.Int("active processes"), + "idle": c.Int("idle processes"), + "max_active": c.Int("max active processes"), + "max_children_reached": c.Int("max children reached"), + "total": c.Int("total processes"), }, - "slow_requests": c.Int("slow requests"), } ) From 4c910e58697ab1e4db42b1439365b9db65645f49 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Tue, 3 Oct 2017 09:15:58 +0200 Subject: [PATCH 64/96] Implement logic for disabled groups (#5295) --- libbeat/kibana/testdata/fields.yml | 7 +++ libbeat/kibana/transform.go | 4 +- libbeat/kibana/transform_test.go | 68 ++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/libbeat/kibana/testdata/fields.yml b/libbeat/kibana/testdata/fields.yml index f8a8c7d835e..76154b7a6e1 100644 --- a/libbeat/kibana/testdata/fields.yml +++ b/libbeat/kibana/testdata/fields.yml @@ -5,6 +5,13 @@ Contains common beat fields available in all event types. fields: + - name: group_disabled + type: group + enabled: false + fields: + - name: message + type: text + - name: beat.name description: > The name of the Beat sending the log messages. If the Beat name is diff --git a/libbeat/kibana/transform.go b/libbeat/kibana/transform.go index 68adb50af58..6b334171efc 100644 --- a/libbeat/kibana/transform.go +++ b/libbeat/kibana/transform.go @@ -59,7 +59,9 @@ func (t *Transformer) transformFields(commonFields common.Fields, path string) { } if f.Type == "group" { - t.transformFields(f.Fields, f.Path) + if f.Enabled == nil || *f.Enabled { + t.transformFields(f.Fields, f.Path) + } } else { // set default values (as done in python script) t.keys[f.Path] = true diff --git a/libbeat/kibana/transform_test.go b/libbeat/kibana/transform_test.go index 7718a0aaa78..6aff5707d32 100644 --- a/libbeat/kibana/transform_test.go +++ b/libbeat/kibana/transform_test.go @@ -283,3 +283,71 @@ func TestTransformFieldFormatMap(t *testing.T) { assert.Equal(t, test.expected, out, fmt.Sprintf("Failed for idx %v", idx)) } } +func TestTransformGroupAndEnabled(t *testing.T) { + tests := []struct { + commonFields common.Fields + expected []string + }{ + { + commonFields: common.Fields{common.Field{Name: "context", Path: "something"}}, + expected: []string{"context"}, + }, + { + commonFields: common.Fields{ + common.Field{ + Name: "context", + Type: "group", + Fields: common.Fields{ + common.Field{Name: "type", Type: ""}, + common.Field{ + Name: "metric", + Type: "group", + Fields: common.Fields{ + common.Field{Name: "object"}, + }, + }, + }, + }, + }, + expected: []string{"context.type", "context.metric.object"}, + }, + { + commonFields: common.Fields{ + common.Field{Name: "enabledField"}, + common.Field{Name: "disabledField", Enabled: &falsy}, //enabled is ignored for Type!=group + common.Field{ + Name: "enabledGroup", + Type: "group", + Enabled: &truthy, + Fields: common.Fields{ + common.Field{Name: "type", Type: ""}, + }, + }, + common.Field{ + Name: "context", + Type: "group", + Enabled: &falsy, + Fields: common.Fields{ + common.Field{Name: "type", Type: ""}, + common.Field{ + Name: "metric", + Type: "group", + Fields: common.Fields{ + common.Field{Name: "object"}, + }, + }, + }, + }, + }, + expected: []string{"enabledField", "disabledField", "enabledGroup.type"}, + }, + } + for idx, test := range tests { + trans := NewTransformer("name", "title", test.commonFields) + out := trans.TransformFields()["fields"].([]common.MapStr) + assert.Equal(t, len(test.expected)+ctMetaData, len(out)) + for i, e := range test.expected { + assert.Equal(t, e, out[i]["name"], fmt.Sprintf("Failed for idx %v", idx)) + } + } +} From 397e2411534ccb828569cd3ee4090bbe299230e5 Mon Sep 17 00:00:00 2001 From: Didave Date: Tue, 3 Oct 2017 04:11:44 -0400 Subject: [PATCH 65/96] Fixed typos in reference config files and added default value for loadbalance (#5238) --- auditbeat/auditbeat.reference.yml | 18 +++++++++--------- filebeat/filebeat.reference.yml | 18 +++++++++--------- heartbeat/heartbeat.reference.yml | 18 +++++++++--------- libbeat/_meta/config.reference.yml | 18 +++++++++--------- libbeat/docs/outputconfig.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 18 +++++++++--------- packetbeat/packetbeat.reference.yml | 18 +++++++++--------- winlogbeat/winlogbeat.reference.yml | 18 +++++++++--------- 8 files changed, 64 insertions(+), 64 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 665125f9fc9..0c4d78055ce 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -303,15 +303,15 @@ output.elasticsearch: # Not yet supported for async connections (i.e. with the "pipelining" option set) #ttl: 30s - # Optional load balance the events between the Logstash hosts - #loadbalance: true + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false # Number of batches to be sent asynchronously to logstash while processing # new batches. #pipelining: 5 - # If enabled only a subset of events in a batch of events is transfered per - # transaction. The number of events to sent increases up to `bulk_max_size` + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` # if no error is encountered. #slow_start: false @@ -647,7 +647,7 @@ output.elasticsearch: #============================== Dashboards ===================================== # These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the +# the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. #setup.dashboards.enabled: false @@ -688,8 +688,8 @@ output.elasticsearch: # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.name: "auditbeat-%{[beat.version]}" -# Template patttern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indicies. +# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.pattern: "auditbeat-%{[beat.version]}-*" @@ -726,7 +726,7 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" - # Optional protocol and basic auth crendentials. + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" @@ -794,7 +794,7 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# Logging to rotating files files. Set logging.to_files to false to disable logging to +# Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true logging.files: diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 72a12bb38b6..d1d93174ae1 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -723,15 +723,15 @@ output.elasticsearch: # Not yet supported for async connections (i.e. with the "pipelining" option set) #ttl: 30s - # Optional load balance the events between the Logstash hosts - #loadbalance: true + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false # Number of batches to be sent asynchronously to logstash while processing # new batches. #pipelining: 5 - # If enabled only a subset of events in a batch of events is transfered per - # transaction. The number of events to sent increases up to `bulk_max_size` + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` # if no error is encountered. #slow_start: false @@ -1067,7 +1067,7 @@ output.elasticsearch: #============================== Dashboards ===================================== # These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the +# the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. #setup.dashboards.enabled: false @@ -1108,8 +1108,8 @@ output.elasticsearch: # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.name: "filebeat-%{[beat.version]}" -# Template patttern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indicies. +# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.pattern: "filebeat-%{[beat.version]}-*" @@ -1146,7 +1146,7 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" - # Optional protocol and basic auth crendentials. + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" @@ -1214,7 +1214,7 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# Logging to rotating files files. Set logging.to_files to false to disable logging to +# Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true logging.files: diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 5b8c2d04741..5e1f7d1e02a 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -452,15 +452,15 @@ output.elasticsearch: # Not yet supported for async connections (i.e. with the "pipelining" option set) #ttl: 30s - # Optional load balance the events between the Logstash hosts - #loadbalance: true + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false # Number of batches to be sent asynchronously to logstash while processing # new batches. #pipelining: 5 - # If enabled only a subset of events in a batch of events is transfered per - # transaction. The number of events to sent increases up to `bulk_max_size` + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` # if no error is encountered. #slow_start: false @@ -796,7 +796,7 @@ output.elasticsearch: #============================== Dashboards ===================================== # These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the +# the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. #setup.dashboards.enabled: false @@ -837,8 +837,8 @@ output.elasticsearch: # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.name: "heartbeat-%{[beat.version]}" -# Template patttern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indicies. +# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.pattern: "heartbeat-%{[beat.version]}-*" @@ -875,7 +875,7 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" - # Optional protocol and basic auth crendentials. + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" @@ -943,7 +943,7 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# Logging to rotating files files. Set logging.to_files to false to disable logging to +# Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true logging.files: diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 5e8eaec312e..fec940b1219 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -238,15 +238,15 @@ output.elasticsearch: # Not yet supported for async connections (i.e. with the "pipelining" option set) #ttl: 30s - # Optional load balance the events between the Logstash hosts - #loadbalance: true + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false # Number of batches to be sent asynchronously to logstash while processing # new batches. #pipelining: 5 - # If enabled only a subset of events in a batch of events is transfered per - # transaction. The number of events to sent increases up to `bulk_max_size` + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` # if no error is encountered. #slow_start: false @@ -582,7 +582,7 @@ output.elasticsearch: #============================== Dashboards ===================================== # These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the +# the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. #setup.dashboards.enabled: false @@ -623,8 +623,8 @@ output.elasticsearch: # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.name: "beat-index-prefix-%{[beat.version]}" -# Template patttern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indicies. +# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.pattern: "beat-index-prefix-%{[beat.version]}-*" @@ -661,7 +661,7 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" - # Optional protocol and basic auth crendentials. + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" @@ -729,7 +729,7 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# Logging to rotating files files. Set logging.to_files to false to disable logging to +# Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true logging.files: diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index b2954bcd2d2..01f4df07368 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -445,7 +445,7 @@ output.logstash: ===== `pipelining` -Configures number of batches to be send asynchronously to logstash while waiting +Configures number of batches to be sent asynchronously to logstash while waiting for ACK from logstash. Output only becomes blocking once number of `pipelining` batches have been written. Pipelining is disabled if a values of 0 is configured. The default value is 0. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 081934a61a1..578d06b4478 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -682,15 +682,15 @@ output.elasticsearch: # Not yet supported for async connections (i.e. with the "pipelining" option set) #ttl: 30s - # Optional load balance the events between the Logstash hosts - #loadbalance: true + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false # Number of batches to be sent asynchronously to logstash while processing # new batches. #pipelining: 5 - # If enabled only a subset of events in a batch of events is transfered per - # transaction. The number of events to sent increases up to `bulk_max_size` + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` # if no error is encountered. #slow_start: false @@ -1026,7 +1026,7 @@ output.elasticsearch: #============================== Dashboards ===================================== # These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the +# the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. #setup.dashboards.enabled: false @@ -1067,8 +1067,8 @@ output.elasticsearch: # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.name: "metricbeat-%{[beat.version]}" -# Template patttern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indicies. +# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.pattern: "metricbeat-%{[beat.version]}-*" @@ -1105,7 +1105,7 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" - # Optional protocol and basic auth crendentials. + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" @@ -1173,7 +1173,7 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# Logging to rotating files files. Set logging.to_files to false to disable logging to +# Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true logging.files: diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 5cb8fadc656..a730f3ca170 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -690,15 +690,15 @@ output.elasticsearch: # Not yet supported for async connections (i.e. with the "pipelining" option set) #ttl: 30s - # Optional load balance the events between the Logstash hosts - #loadbalance: true + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false # Number of batches to be sent asynchronously to logstash while processing # new batches. #pipelining: 5 - # If enabled only a subset of events in a batch of events is transfered per - # transaction. The number of events to sent increases up to `bulk_max_size` + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` # if no error is encountered. #slow_start: false @@ -1034,7 +1034,7 @@ output.elasticsearch: #============================== Dashboards ===================================== # These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the +# the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. #setup.dashboards.enabled: false @@ -1075,8 +1075,8 @@ output.elasticsearch: # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.name: "packetbeat-%{[beat.version]}" -# Template patttern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indicies. +# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.pattern: "packetbeat-%{[beat.version]}-*" @@ -1113,7 +1113,7 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" - # Optional protocol and basic auth crendentials. + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" @@ -1181,7 +1181,7 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# Logging to rotating files files. Set logging.to_files to false to disable logging to +# Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true logging.files: diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 79b1b469c22..0a592543f46 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -267,15 +267,15 @@ output.elasticsearch: # Not yet supported for async connections (i.e. with the "pipelining" option set) #ttl: 30s - # Optional load balance the events between the Logstash hosts - #loadbalance: true + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false # Number of batches to be sent asynchronously to logstash while processing # new batches. #pipelining: 5 - # If enabled only a subset of events in a batch of events is transfered per - # transaction. The number of events to sent increases up to `bulk_max_size` + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` # if no error is encountered. #slow_start: false @@ -611,7 +611,7 @@ output.elasticsearch: #============================== Dashboards ===================================== # These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the +# the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. #setup.dashboards.enabled: false @@ -652,8 +652,8 @@ output.elasticsearch: # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.name: "winlogbeat-%{[beat.version]}" -# Template patttern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indicies. +# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. #setup.template.pattern: "winlogbeat-%{[beat.version]}-*" @@ -690,7 +690,7 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" - # Optional protocol and basic auth crendentials. + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" @@ -758,7 +758,7 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# Logging to rotating files files. Set logging.to_files to false to disable logging to +# Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true logging.files: From 88680a800eafd7479ceeec2c330293a15daa6f80 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Tue, 3 Oct 2017 10:33:53 +0200 Subject: [PATCH 66/96] Add multifields support. (#5296) --- .../kibana/index_pattern_generator_test.go | 29 +- libbeat/kibana/testdata/beat-5x.json | 6 + libbeat/kibana/testdata/beat-default.json | 16 + libbeat/kibana/testdata/extensive/fields.yml | 6858 ++++++++++++++++ .../metricbeat-5x.json} | 0 .../metricbeat-default.json} | 0 libbeat/kibana/testdata/fields.yml | 6863 +---------------- libbeat/kibana/transform.go | 9 + libbeat/kibana/transform_test.go | 20 + 9 files changed, 6942 insertions(+), 6859 deletions(-) create mode 100644 libbeat/kibana/testdata/beat-5x.json create mode 100644 libbeat/kibana/testdata/beat-default.json create mode 100644 libbeat/kibana/testdata/extensive/fields.yml rename libbeat/kibana/testdata/{metricbeat-5x-old.json => extensive/metricbeat-5x.json} (100%) rename libbeat/kibana/testdata/{metricbeat-default-old.json => extensive/metricbeat-default.json} (100%) diff --git a/libbeat/kibana/index_pattern_generator_test.go b/libbeat/kibana/index_pattern_generator_test.go index 54d26938c52..a450315b57f 100644 --- a/libbeat/kibana/index_pattern_generator_test.go +++ b/libbeat/kibana/index_pattern_generator_test.go @@ -97,16 +97,37 @@ func TestDumpToFileDefault(t *testing.T) { func TestGenerate(t *testing.T) { beatDir := tmpPath() defer teardown(beatDir) - generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1") + generator, err := NewGenerator("beat-*", "b eat ?!", beatDir, "7.0.0-alpha1") + pattern, err := generator.Generate() + assert.NoError(t, err) + assert.Equal(t, 2, len(pattern)) + + tests := []map[string]string{ + {"existing": "beat-5x.json", "created": "_meta/kibana/5.x/index-pattern/beat.json"}, + {"existing": "beat-default.json", "created": "_meta/kibana/default/index-pattern/beat.json"}, + } + testGenerate(t, beatDir, tests) +} + +func TestGenerateExtensive(t *testing.T) { + beatDir, err := filepath.Abs("./testdata/extensive") + if err != nil { + panic(err) + } + defer teardown(beatDir) + generator, err := NewGenerator("metricbeat-*", "metric be at ?!", beatDir, "7.0.0-alpha1") pattern, err := generator.Generate() assert.NoError(t, err) assert.Equal(t, 2, len(pattern)) tests := []map[string]string{ - {"existing": "metricbeat-5x-old.json", "created": "_meta/kibana/5.x/index-pattern/metricbeat.json"}, - {"existing": "metricbeat-default-old.json", "created": "_meta/kibana/default/index-pattern/metricbeat.json"}, + {"existing": "metricbeat-5x.json", "created": "_meta/kibana/5.x/index-pattern/metricbeat.json"}, + {"existing": "metricbeat-default.json", "created": "_meta/kibana/default/index-pattern/metricbeat.json"}, } + testGenerate(t, beatDir, tests) +} +func testGenerate(t *testing.T, beatDir string, tests []map[string]string) { for _, test := range tests { // compare default existing, err := readJson(filepath.Join(beatDir, test["existing"])) @@ -147,7 +168,7 @@ func TestGenerate(t *testing.T) { assert.NoError(t, err) err = json.Unmarshal([]byte(attrCreated["fields"].(string)), &fieldsCreated) assert.NoError(t, err) - assert.Equal(t, len(ffmExisting), len(ffmCreated)) + assert.Equal(t, len(fieldsExisting), len(fieldsCreated)) for _, e := range fieldsExisting { idx := find(fieldsCreated, e["name"].(string)) assert.NotEqual(t, -1, idx) diff --git a/libbeat/kibana/testdata/beat-5x.json b/libbeat/kibana/testdata/beat-5x.json new file mode 100644 index 00000000000..4a286691010 --- /dev/null +++ b/libbeat/kibana/testdata/beat-5x.json @@ -0,0 +1,6 @@ +{ + "fieldFormatMap": "{}", + "fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"long\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field.keyword\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"}]", + "timeFieldName": "@timestamp", + "title": "beat-*" +} \ No newline at end of file diff --git a/libbeat/kibana/testdata/beat-default.json b/libbeat/kibana/testdata/beat-default.json new file mode 100644 index 00000000000..387e955d2ff --- /dev/null +++ b/libbeat/kibana/testdata/beat-default.json @@ -0,0 +1,16 @@ +{ + "objects": [ + { + "attributes": { + "fieldFormatMap": "{}", + "fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"long\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field.keyword\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"}]", + "timeFieldName": "@timestamp", + "title": "beat-*" + }, + "id": "beat-*", + "type": "index-pattern", + "version": 1 + } + ], + "version": "7.0.0-alpha1" +} \ No newline at end of file diff --git a/libbeat/kibana/testdata/extensive/fields.yml b/libbeat/kibana/testdata/extensive/fields.yml new file mode 100644 index 00000000000..f8a8c7d835e --- /dev/null +++ b/libbeat/kibana/testdata/extensive/fields.yml @@ -0,0 +1,6858 @@ + +- key: beat + title: Beat + description: > + Contains common beat fields available in all event types. + fields: + + - name: beat.name + description: > + The name of the Beat sending the log messages. If the Beat name is + set in the configuration file, then that value is used. If it is not + set, the hostname is used. To set the Beat name, use the `name` + option in the configuration file. + - name: beat.hostname + description: > + The hostname as returned by the operating system on which the Beat is + running. + - name: beat.timezone + description: > + The timezone as returned by the operating system on which the Beat is + running. + - name: beat.version + description: > + The version of the beat that generated this event. + + - name: "@timestamp" + type: date + required: true + format: date + example: August 26th 2016, 12:35:53.332 + description: > + The timestamp when the event log record was generated. + + - name: tags + description: > + Arbitrary tags that can be set per Beat and per transaction + type. + + - name: fields + type: object + object_type: keyword + description: > + Contains user configurable fields. + + - name: error + type: group + description: > + Error fields containing additional info in case of errors. + fields: + - name: message + type: text + description: > + Error message. + - name: code + type: long + description: > + Error code. + - name: type + type: keyword + description: > + Error type. +- key: cloud + title: Cloud provider metadata + description: > + Metadata from cloud providers added by the add_cloud_metadata processor. + fields: + + - name: meta.cloud.provider + example: ec2 + description: > + Name of the cloud provider. Possible values are ec2, gce, or digitalocean. + + - name: meta.cloud.instance_id + description: > + Instance ID of the host machine. + + - name: meta.cloud.instance_name + description: > + Instance name of the host machine. + + - name: meta.cloud.machine_type + example: t2.medium + description: > + Machine type of the host machine. + + - name: meta.cloud.availability_zone + example: us-east-1c + description: > + Availability zone in which this host is running. + + - name: meta.cloud.project_id + example: project-x + description: > + Name of the project in Google Cloud. + + - name: meta.cloud.region + description: > + Region in which this host is running. +- key: docker + title: Docker + description: > + beta[] + + Docker stats collected from Docker. + short_config: false + anchor: docker-processor + fields: + - name: docker + type: group + fields: + - name: container.id + type: keyword + description: > + Unique container id. + - name: container.image + type: keyword + description: > + Name of the image the container was built on. + - name: container.name + type: keyword + description: > + Container name. + - name: container.labels + type: object + object_type: keyword + description: > + Image labels. +- key: kubernetes + title: Kubernetes + description: > + beta[] + + Kubernetes metadata added by the kubernetes processor + short_config: false + anchor: kubernetes-processor + fields: + - name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: labels + type: object + description: > + Kubernetes labels map + + - name: annotations + type: object + description: > + Kubernetes annotations map + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: > + Kubernetes container image +- key: common + title: Common + description: > + Contains common fields available in all event types. + fields: + + - name: metricset.module + description: > + The name of the module that generated the event. + + - name: metricset.name + description: > + The name of the metricset that generated the event. + + - name: metricset.host + description: > + Hostname of the machine from which the metricset was collected. This + field may not be present when the data was collected locally. + + - name: metricset.rtt + type: long + required: true + description: > + Event round trip time in microseconds. + + - name: metricset.namespace + type: keyword + description: > + Namespace of dynamic metricsets. + + - name: type + required: true + example: metricsets + description: > + The document type. Always set to "metricsets". + +- key: aerospike + title: "Aerospike" + description: > + experimental[] + + Aerospike module + fields: + - name: aerospike + type: group + description: > + fields: + - name: namespace + type: group + description: > + namespace + fields: + - name: client + type: group + description: > + Client stats. + fields: + - name: delete + type: group + description: > + Client delete transactions stats. + fields: + - name: error + type: long + description: > + Number of client delete transactions that failed with an error. + - name: not_found + type: long + description: > + Number of client delete transactions that resulted in a not found. + - name: success + type: long + description: > + Number of successful client delete transactions. + - name: timeout + type: long + description: > + Number of client delete transactions that timed out. + - name: read + type: group + description: > + Client read transactions stats. + fields: + - name: error + type: long + description: > + Number of client read transaction errors. + - name: not_found + type: long + description: > + Number of client read transaction that resulted in not found. + - name: success + type: long + description: > + Number of successful client read transactions. + - name: timeout + type: long + description: > + Number of client read transaction that timed out. + - name: write + type: group + description: > + Client write transactions stats. + fields: + - name: error + type: long + description: > + Number of client write transactions that failed with an error. + - name: success + type: long + description: > + Number of successful client write transactions. + - name: timeout + type: long + description: > + Number of client write transactions that timed out. + - name: device + type: group + description: > + Disk storage stats + fields: + - name: available.pct + type: scaled_float + format: percent + description: > + Measures the minimum contiguous disk space across all disks in a namespace. + - name: free.pct + type: scaled_float + format: percent + description: > + Percentage of disk capacity free for this namespace. + - name: total.bytes + type: long + format: bytes + description: > + Total bytes of disk space allocated to this namespace on this node. + - name: used.bytes + type: long + format: bytes + description: > + Total bytes of disk space used by this namespace on this node. + - name: hwm_breached + type: boolean + description: > + If true, Aerospike has breached 'high-water-[disk|memory]-pct' for this namespace. + - name: memory + type: group + description: > + Memory storage stats. + fields: + - name: free.pct + type: scaled_float + format: percent + description: > + Percentage of memory capacity free for this namespace on this node. + - name: used.data.bytes + type: long + format: bytes + description: > + Amount of memory occupied by data for this namespace on this node. + - name: used.index.bytes + type: long + format: bytes + description: > + Amount of memory occupied by the index for this namespace on this node. + - name: used.sindex.bytes + type: long + format: bytes + description: > + Amount of memory occupied by secondary indexes for this namespace on this node. + - name: used.total.bytes + type: long + format: bytes + description: > + Total bytes of memory used by this namespace on this node. + - name: name + type: keyword + description: > + Namespace name + - name: node.host + type: keyword + desription: > + Node host + - name: node.name + type: keyword + description: > + Node name + - name: objects + type: group + description: > + Records stats. + fields: + - name: master + type: long + description: > + Number of records on this node which are active masters. + - name: total + type: long + description: > + Number of records in this namespace for this node. + - name: stop_writes + type: boolean + description: > + If true this namespace is currently not allowing writes. + +- key: apache + title: "Apache" + description: > + Apache HTTPD server metricsets collected from the Apache web server. + short_config: false + fields: + - name: apache + type: group + description: > + `apache` contains the metrics that were scraped from Apache. + fields: + - name: status + type: group + description: > + `status` contains the metrics that were scraped from the Apache status page. + fields: + - name: hostname + type: keyword + description: > + Apache hostname. + - name: total_accesses + type: long + description: > + Total number of access requests. + - name: total_kbytes + type: long + description: > + Total number of kilobytes served. + - name: requests_per_sec + type: scaled_float + description: > + Requests per second. + - name: bytes_per_sec + type: scaled_float + description: > + Bytes per second. + - name: bytes_per_request + type: scaled_float + description: > + Bytes per request. + - name: workers.busy + type: long + description: > + Number of busy workers. + - name: workers.idle + type: long + description: > + Number of idle workers. + - name: uptime + type: group + description: > + Uptime stats. + fields: + - name: server_uptime + type: long + description: > + Server uptime in seconds. + - name: uptime + type: long + description: > + Server uptime. + - name: cpu + type: group + description: > + CPU stats. + fields: + - name: load + type: scaled_float + description: > + CPU Load. + - name: user + type: scaled_float + description: > + CPU user load. + - name: system + type: scaled_float + description: > + System cpu. + - name: children_user + type: scaled_float + description: > + CPU of children user. + - name: children_system + type: scaled_float + description: > + CPU of children system. + - name: connections + type: group + description: > + Connection stats. + fields: + - name: total + type: long + description: > + Total connections. + - name: async.writing + type: long + description: > + Async connection writing. + - name: async.keep_alive + type: long + description: > + Async keeped alive connections. + - name: async.closing + type: long + description: > + Async closed connections. + - name: load + type: group + description: > + Load averages. + fields: + - name: "1" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last minute. + - name: "5" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 5 minutes. + - name: "15" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 15 minutes. + - name: scoreboard + type: group + description: > + Scoreboard metrics. + fields: + - name: starting_up + type: long + description: > + Starting up. + - name: reading_request + type: long + description: > + Reading requests. + - name: sending_reply + type: long + description: > + Sending Reply. + - name: keepalive + type: long + description: > + Keep alive. + - name: dns_lookup + type: long + description: > + Dns Lookups. + - name: closing_connection + type: long + description: > + Closing connections. + - name: logging + type: long + description: > + Logging + - name: gracefully_finishing + type: long + description: > + Gracefully finishing. + - name: idle_cleanup + type: long + description: > + Idle cleanups. + - name: open_slot + type: long + description: > + Open slots. + - name: waiting_for_connection + type: long + description: > + Waiting for connections. + - name: total + type: long + description: > + Total. + +- key: ceph + title: "Ceph" + description: > + beta[] + + Ceph module + short_config: false + fields: + - name: ceph + type: group + description: > + `ceph` contains the metrics that were scraped from CEPH. + fields: + - name: cluster_disk + type: group + description: > + cluster_disk + fields: + - name: available.bytes + type: long + description: > + Available bytes of the cluster + format: bytes + - name: total.bytes + type: long + description: > + Total bytes of the cluster + format: bytes + - name: used.bytes + type: long + description: > + Used bytes of the cluster + format: bytes + + - name: cluster_health + type: group + description: > + cluster_health + fields: + - name: overall_status + type: keyword + description: > + Overall status of the cluster + - name: timechecks.epoch + type: long + description: > + Map version + - name: timechecks.round.value + type: long + description: > + timecheck round + - name: timechecks.round.status + type: keyword + description: > + Status of the round + + - name: cluster_status + type: group + description: > + cluster_status + fields: + - name: version + type: long + description: > + Ceph Status version + - name: traffic.read_bytes + type: long + format: bytes + description: > + Cluster read throughput per second + - name: traffic.write_bytes + type: long + format: bytes + description: > + Cluster write throughput per second + - name: traffic.read_op_per_sec + type: long + description: > + Cluster read iops per second + - name: traffic.write_op_per_sec + type: long + description: > + Cluster write iops per second + - name: misplace.total + type: long + description: > + Cluster misplace pg number + - name: misplace.objects + type: long + description: > + Cluster misplace objects number + - name: misplace.ratio + type: scaled_float + format: percent + description: > + Cluster misplace ratio + - name: degraded.total + type: long + description: > + Cluster degraded pg number + - name: degraded.objects + type: long + description: > + Cluster degraded objects number + - name: degraded.ratio + type: scaled_float + format: percent + description: > + Cluster degraded ratio + - name: pg.data_bytes + type: long + format: bytes + description: > + Cluster pg data bytes + - name: pg.avail_bytes + type: long + format: bytes + description: > + Cluster available bytes + - name: pg.total_bytes + type: long + format: bytes + description: > + Cluster total bytes + - name: pg.used_bytes + type: long + format: bytes + description: > + Cluster used bytes + - name: pg_state.state_name + type: long + description: > + Pg state description + - name: pg_state.count + type: long + description: > + Shows how many pgs are in state of pg_state.state_name + - name: pg_state.version + type: long + description: > + Cluster status version + - name: osd.full + type: boolean + description: > + Is osd full + - name: osd.nearfull + type: boolean + description: > + Is osd near full + - name: osd.num_osds + type: long + description: > + Shows how many osds in the cluster + - name: osd.num_up_osds + type: long + description: > + Shows how many osds are on the state of UP + - name: osd.num_in_osds + type: long + description: > + Shows how many osds are on the state of IN + - name: osd.num_in_osds2 + type: long + description: > + Shows how many osds are on the state of REMAPPED + - name: osd.epoch + type: long + description: > + epoch number + + - name: monitor_health + type: group + description: > + monitor_health stats data + fields: + - name: available.pct + type: long + description: > + Available percent of the MON + - name: health + type: keyword + description: > + Health of the MON + - name: available.kb + type: long + description: > + Available KB of the MON + - name: total.kb + type: long + description: > + Total KB of the MON + - name: used.kb + type: long + description: > + Used KB of the MON + - name: last_updated + type: date + description: > + Time when was updated + - name: name + type: keyword + description: > + Name of the MON + - name: store_stats.log.bytes + type: long + description: > + Log bytes of MON + format: bytes + - name: store_stats.misc.bytes + type: long + description: > + Misc bytes of MON + format: bytes + - name: store_stats.sst.bytes + type: long + description: > + SST bytes of MON + format: bytes + - name: store_stats.total.bytes + type: long + description: > + Total bytes of MON + format: bytes + - name: store_stats.last_updated + type: long + description: > + Last updated + + - name: pool_disk + type: group + description: > + pool_disk + fields: + - name: id + type: long + description: > + Id of the pool + - name: name + type: keyword + description: > + Name of the pool + - name: stats.available.bytes + type: long + description: > + Available bytes of the pool + format: bytes + - name: stats.objects + type: long + description: > + Number of objects of the pool + - name: stats.used.bytes + type: long + description: > + Used bytes of the pool + format: bytes + - name: stats.used.kb + type: long + description: > + Used kb of the pool + +- key: couchbase + title: "Couchbase" + description: > + beta[] + + Metrics collected from Couchbase servers. + short_config: false + fields: + - name: couchbase + type: group + description: > + `couchbase` contains the metrics that were scraped from Couchbase. + fields: + - name: bucket + type: group + description: > + Couchbase bucket metrics. + fields: + - name: name + type: keyword + description: > + Name of the bucket. + - name: type + type: keyword + description: > + Type of the bucket. + - name: data.used.bytes + format: bytes + type: long + description: > + Size of user data within buckets of the specified state that are resident in RAM. + - name: disk.fetches + type: long + description: > + Number of disk fetches. + - name: disk.used.bytes + format: bytes + type: long + description: > + Amount of disk used (bytes). + - name: memory.used.bytes + format: bytes + type: long + description: > + Amount of memory used by the bucket (bytes). + - name: quota.ram.bytes + format: bytes + type: long + description: > + Amount of RAM used by the bucket (bytes). + - name: quota.use.pct + format: percent + type: scaled_float + description: > + Percentage of RAM used (for active objects) against the configured bucket size (%). + - name: ops_per_sec + type: long + description: > + Number of operations per second. + - name: item_count + type: long + description: > + Number of items associated with the bucket. + + - name: cluster + type: group + description: > + Couchbase cluster metrics. + fields: + - name: hdd.free.bytes + format: bytes + type: long + description: > + Free hard drive space in the cluster (bytes). + - name: hdd.quota.total.bytes + format: bytes + type: long + description: > + Hard drive quota total for the cluster (bytes). + - name: hdd.total.bytes + format: bytes + type: long + description: > + Total hard drive space available to the cluster (bytes). + - name: hdd.used.value.bytes + format: bytes + type: long + description: > + Hard drive space used by the cluster (bytes). + - name: hdd.used.by_data.bytes + format: bytes + type: long + description: > + Hard drive space used by the data in the cluster (bytes). + - name: max_bucket_count + type: long + description: > + Max bucket count setting. + - name: quota.index_memory.mb + type: long + description: > + Memory quota setting for the Index service (Mbyte). + - name: quota.memory.mb + type: long + description: > + Memory quota setting for the cluster (Mbyte). + - name: ram.quota.total.value.bytes + format: bytes + type: long + description: > + RAM quota total for the cluster (bytes). + - name: ram.quota.total.per_node.bytes + format: bytes + type: long + description: > + RAM quota used by the current node in the cluster (bytes). + - name: ram.quota.used.value.bytes + format: bytes + type: long + description: > + RAM quota used by the cluster (bytes). + - name: ram.quota.used.per_node.bytes + format: bytes + type: long + description: > + Ram quota used by the current node in the cluster (bytes) + - name: ram.total.bytes + format: bytes + type: long + description: > + Total RAM available to cluster (bytes). + - name: ram.used.value.bytes + format: bytes + type: long + description: > + RAM used by the cluster (bytes). + - name: ram.used.by_data.bytes + format: bytes + type: long + description: > + RAM used by the data in the cluster (bytes). + + - name: node + type: group + description: > + Couchbase node metrics. + fields: + - name: cmd_get + type: long + description: > + Number of get commands + - name: couch.docs.disk_size.bytes + format: bytes + type: long + description: > + Amount of disk space used by Couch docs (bytes). + - name: couch.docs.data_size.bytes + format: bytes + type: long + description: > + Data size of Couch docs associated with a node (bytes). + - name: couch.spatial.data_size.bytes + type: long + description: > + Size of object data for spatial views (bytes). + - name: couch.spatial.disk_size.bytes + type: long + description: > + Amount of disk space used by spatial views (bytes). + - name: couch.views.disk_size.bytes + type: long + description: > + Amount of disk space used by Couch views (bytes). + - name: couch.views.data_size.bytes + type: long + description: > + Size of object data for Couch views (bytes). + - name: cpu_utilization_rate.pct + type: scaled_float + description: > + The CPU utilization rate (%). + - name: current_items.value + type: long + description: > + Number of current items. + - name: current_items.total + type: long + description: > + Total number of items associated with the node. + - name: ep_bg_fetched + type: long + description: > + Number of disk fetches performed since the server was started. + - name: get_hits + type: long + description: > + Number of get hits. + - name: hostname + type: keyword + description: > + The hostname of the node. + - name: mcd_memory.allocated.bytes + format: bytes + type: long + description: > + Amount of memcached memory allocated (bytes). + - name: mcd_memory.reserved.bytes + type: long + description: > + Amount of memcached memory reserved (bytes). + - name: memory.free.bytes + type: long + description: > + Amount of memory free for the node (bytes). + - name: memory.total.bytes + type: long + description: > + Total memory available to the node (bytes). + - name: memory.used.bytes + type: long + description: > + Memory used by the node (bytes). + - name: ops + type: long + description: > + Number of operations performed on Couchbase. + - name: swap.total.bytes + type: long + description: > + Total swap size allocated (bytes). + - name: swap.used.bytes + type: long + description: > + Amount of swap space used (bytes). + - name: uptime.sec + type: long + description: > + Time during which the node was in operation (sec). + - name: vb_replica_curr_items + type: long + description: > + Number of items/documents that are replicas. + +- key: docker + title: "Docker" + description: > + beta[] + + Docker stats collected from Docker. + short_config: false + fields: + - name: docker + type: group + description: > + Information and statistics about docker's running containers. + fields: + - name: container + type: group + description: > + Docker container metrics. + fields: + - name: command + type: keyword + description: > + Command that was executed in the Docker container. + - name: created + type: date + description: > + Date when the container was created. + - name: status + type: keyword + description: > + Container status. + - name: size + type: group + description: > + Container size metrics. + fields: + - name: root_fs + type: long + description: > + Total size of all the files in the container. + - name: rw + type: long + description: > + Size of the files that have been created or changed since creation. + - name: tags + type: array + description: > + Image tags. + + - name: cpu + type: group + description: > + Runtime CPU metrics. + fields: + - name: kernel.pct + type: scaled_float + format: percentage + description: > + The system kernel consumed by the Docker server. + - name: kernel.ticks + type: long + description: > + CPU kernel ticks. + - name: system.pct + type: scaled_float + format: percentage + description: > + - name: system.ticks + type: long + description: > + CPU system ticks. + - name: user.pct + type: scaled_float + format: percentage + description: > + - name: user.ticks + type: long + description: > + CPU user ticks + - name: total.pct + type: scaled_float + format: percentage + description: > + Total CPU usage. + # TODO: how to document cpu list? + #- name: core + # type: array + # description: > + # Dictionary with list of cpu and usage inside. + + - name: diskio + type: group + description: > + Disk I/O metrics. + fields: + - name: reads + type: scaled_float + description: > + Number of reads. + - name: writes + type: scaled_float + description: > + Number of writes. + - name: total + type: scaled_float + description: > + Number of reads and writes combined. + + - name: healthcheck + type: group + description: > + Docker container metrics. + fields: + - name: failingstreak + type: integer + description: > + concurent failed check + - name: status + type: keyword + description: > + Healthcheck status code + - name: event + type: group + description: > + event fields. + fields: + - name: end_date + type: date + description: > + Healthcheck end date + - name: start_date + type: date + description: > + Healthcheck start date + - name: output + type: keyword + description: > + Healthcheck output + - name: exit_code + type: integer + description: > + Healthcheck status code + + - name: image + type: group + description: > + Docker image metrics. + fields: + - name: id + type: group + description: > + The image layers identifier. + fields: + - name: current + type: keyword + description: > + Unique image identifier given upon its creation. + - name: parent + type: keyword + description: > + Identifier of the image, if it exists, from which the current image directly descends. + - name: created + type: date + description: > + Date and time when the image was created. + - name: size + type: group + description: > + Image size layers. + fields: + - name: virtual + type: long + description: > + Size of the image. + - name: regular + type: long + description: > + Total size of the all cached images associated to the current image. + + - name: labels + type: object + object_type: keyword + description: > + Image labels. + + - name: tags + type: array + description: > + Image tags. + + - name: info + type: group + description: > + beta[] + + Info metrics based on https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/display-system-wide-information. + fields: + - name: containers + type: group + description: > + Overall container stats. + fields: + - name: paused + type: long + description: > + Total number of paused containers. + - name: running + type: long + description: > + Total number of running containers. + - name: stopped + type: long + description: > + Total number of stopped containers. + - name: total + type: long + description: > + Total number of existing containers. + - name: id + type: keyword + description: > + Unique Docker host identifier. + + - name: images + type: long + description: > + Total number of existing images. + + - name: memory + type: group + description: > + Memory metrics. + fields: + + - name: fail.count + type: scaled_float + description: > + Fail counter. + - name: limit + type: long + format: bytes + description: > + Memory limit. + - name: rss + type: group + description: > + RSS memory stats. + fields: + - name: total + type: long + format: bytes + description: > + Total memory resident set size. + - name: pct + type: scaled_float + format: percentage + description: > + Memory resident set size percentage. + - name: usage + type: group + description: > + Usage memory stats. + fields: + - name: max + type: long + format: bytes + description: > + Max memory usage. + - name: pct + type: scaled_float + format: percentage + description: > + Memory usage percentage. + - name: total + type: long + format: bytes + description: > + Total memory usage. + + - name: network + type: group + description: > + Network metrics. + fields: + + - name: interface + type: keyword + description: > + Network interface name. + - name: in + type: group + description: > + Incoming network stats. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of incoming bytes. + - name: dropped + type: scaled_float + description: > + Total number of dropped incoming packets. + - name: errors + type: long + description: > + Total errors on incoming packets. + - name: packets + type: long + description: > + Total number of incoming packets. + - name: out + type: group + description: > + Outgoing network stats. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of outgoing bytes. + - name: dropped + type: scaled_float + description: > + Total number of dropped outgoing packets. + - name: errors + type: long + description: > + Total errors on outgoing packets. + - name: packets + type: long + description: > + Total number of outgoing packets. + +- key: dropwizard + title: "Dropwizard" + description: > + beta[] + + Stats collected from Dropwizard. + short_config: false + fields: + - name: dropwizard + type: group + description: > + fields: + +- key: elasticsearch + title: "Elasticsearch" + description: > + experimental[] + + Elasticsearch module + short_config: false + fields: + - name: elasticsearch + type: group + description: > + fields: + - name: cluster.name + type: keyword + description: > + Elasticsearch cluster name. + - name: node + type: group + description: > + node + fields: + - name: jvm.memory.heap_init.bytes + type: long + format: bytes + description: > + Heap init used by the JVM in bytes. + - name: jvm.version + type: keyword + description: > + JVM version. + - name: name + type: keyword + description: > + Node name. + - name: version + type: keyword + description: > + Node version. + + - name: node.stats + type: group + description: > + node_stats + fields: + - name: indices + type: group + description: > + Node indices stats + fields: + - name: docs.count + type: long + description: > + Total number of existing documents. + - name: docs.deleted + type: long + description: > + Total number of deleted documents. + - name: segments.count + type: long + description: > + Total number of segments. + - name: segments.memory.bytes + type: long + format: bytes + description: > + Total size of segments in bytes. + - name: store.size.bytes + type: long + description: > + Total size of the store in bytes. + - name: jvm.mem.pools + type: group + description: > + JVM memory pool stats + fields: + - name: old + type: group + description: > + Old memory pool stats. + fields: + - name: max.bytes + type: long + format: bytes + description: + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: + Used bytes. + - name: young + type: group + description: > + Young memory pool stats. + fields: + - name: max.bytes + type: long + format: bytes + description: + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: + Used bytes. + - name: survivor + type: group + description: > + Survivor memory pool stats. + fields: + - name: max.bytes + type: long + format: bytes + description: + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: + Used bytes. + + - name: jvm.gc.collectors + type: group + description: > + GC collector stats. + fields: + - name: old.collection + type: group + description: > + Old collection gc. + fields: + - name: count + type: long + description: > + - name: ms + type: long + description: > + - name: young.collection + type: group + description: > + Young collection gc. + fields: + - name: count + type: long + description: > + - name: ms + type: long + description: > + +- key: golang + title: "Golang" + description: > + Golang module + short_config: false + fields: + - name: golang + type: group + description: > + fields: + - name: expvar + type: group + description: > + expvar + fields: + - name: cmdline + type: keyword + description: > + The cmdline of this golang program start with. + - name: heap + type: group + description: > + The golang program heap information exposed by expvar. + fields: + - name: cmdline + type: keyword + description: > + The cmdline of this golang program start with. + + - name: gc + type: group + description: > + Garbage collector summary. + fields: + - name: total_pause + type: group + description: > + Total GC pause duration over lifetime of process. + fields: + - name: ns + type: long + description: > + Duration in Ns. + - name: total_count + type: long + description: > + Total number of GC was happened. + - name: next_gc_limit + type: long + format: bytes + description: > + Next collection will happen when HeapAlloc > this amount. + - name: cpu_fraction + type: long + description: > + Fraction of CPU time used by GC. + - name: pause + type: group + description: > + Last GC pause durations during the monitoring period. + fields: + - name: count + type: long + description: > + Count of GC pause duration during this collect period. + - name: sum + type: group + description: > + Total GC pause duration during this collect period. + fields: + - name: ns + type: long + description: > + Duration in Ns. + - name: max + type: group + description: > + Max GC pause duration during this collect period. + fields: + - name: ns + type: long + description: > + Duration in Ns. + - name: avg + type: group + description: > + Average GC pause duration during this collect period. + fields: + - name: ns + type: long + description: > + Duration in Ns. + + - name: system + type: group + description: > + Heap summary,which bytes was obtained from system. + fields: + - name: total + type: long + format: bytes + description: > + Total bytes obtained from system (sum of XxxSys below). + - name: optained + type: long + format: bytes + description: > + Via HeapSys, bytes obtained from system. heap_sys = heap_idle + heap_inuse. + - name: stack + type: long + format: bytes + description: > + Bytes used by stack allocator, and these bytes was obtained from system. + - name: released + type: long + format: bytes + description: > + Bytes released to the OS. + + - name: allocations + type: group + description: > + Heap allocations summary. + fields: + - name: mallocs + type: long + description: > + Number of mallocs. + - name: frees + type: long + description: > + Number of frees. + - name: objects + type: long + description: > + Total number of allocated objects. + - name: total + type: long + format: bytes + description: > + Bytes allocated (even if freed) throughout the lifetime. + - name: allocated + type: long + format: bytes + description: > + Bytes allocated and not yet freed (same as Alloc above). + - name: idle + type: long + format: bytes + description: > + Bytes in idle spans. + - name: active + type: long + format: bytes + description: > + Bytes in non-idle span. +- key: graphite + title: "graphite" + description: > + []experimental + + graphite Module + fields: + - name: graphite + type: group + description: > + fields: + - name: server + type: group + description: > + server + fields: + - name: example + type: keyword + description: > + Example field + +- key: haproxy + title: "HAProxy" + description: > + HAProxy Module + short_config: false + fields: + - name: haproxy + type: group + description: > + HAProxy metrics. + fields: + - name: info + type: group + description: > + General information about HAProxy processes. + fields: + - name: processes + type: long + description: > + Number of processes. + + - name: process_num + type: long + description: > + Process number. + + - name: pid + type: long + description: > + Process ID. + + - name: run_queue + type: long + description: > + + - name: tasks + type: long + description: > + + - name: uptime.sec + type: long + description: > + Current uptime in seconds. + + - name: memory.max.bytes + type: long + format: bytes + description: > + Maximum amount of memory usage in bytes (the 'Memmax_MB' value converted to bytes). + + - name: ulimit_n + type: long + description: > + Maximum number of open files for the process. + + - name: compress + type: group + description: > + + fields: + - name: bps + type: group + description: > + + fields: + - name: in + type: long + description: > + + - name: out + type: long + description: > + + - name: rate_limit + type: long + description: > + + - name: connection + type: group + description: > + + fields: + - name: rate + type: group + description: > + + fields: + - name: value + type: long + description: > + + - name: limit + type: long + description: > + + - name: max + type: long + description: > + + - name: current + type: long + description: > + Current connections. + + - name: total + type: long + description: > + Total connections. + + - name: ssl.current + type: long + description: > + Current SSL connections. + + - name: ssl.total + type: long + description: > + Total SSL connections. + + - name: ssl.max + type: long + description: > + Maximum SSL connections. + + - name: max + type: long + description: > + Maximum connections. + + - name: hard_max + type: long + description: > + + - name: requests.total + type: long + description: > + + - name: sockets.max + type: long + description: > + + - name: requests.max + type: long + description: > + + - name: pipes + type: group + description: > + fields: + - name: used + type: integer + description: > + + - name: free + type: integer + description: > + + - name: max + type: integer + description: > + + - name: session + type: group + description: + fields: + - name: rate.value + type: integer + description: > + + - name: rate.limit + type: integer + description: > + + - name: rate.max + type: integer + description: > + + + - name: ssl + type: group + description: + fields: + - name: rate.value + type: integer + description: + + - name: rate.limit + type: integer + description: + + - name: rate.max + type: integer + description: + + - name: frontend + type: group + description: + fields: + - name: key_rate.value + type: integer + description: + + - name: key_rate.max + type: integer + description: + + - name: session_reuse.pct + type: scaled_float + format: percent + description: + + - name: backend + type: group + description: + fields: + - name: key_rate.value + type: integer + description: + + - name: key_rate.max + type: integer + description: MaxConnRate + - name: cached_lookups + type: long + description: + - name: cache_misses + type: long + description: + + + - name: zlib_mem_usage + type: group + description: > + + fields: + - name: value + type: integer + description: > + + - name: max + type: integer + description: > + + - name: idle.pct + type: scaled_float + format: percent + description: > + + - name: stat + type: group + description: > + Stats collected from HAProxy processes. + fields: + + - name: status + type: keyword + description: > + Status (UP, DOWN, NOLB, MAINT, or MAINT(via)...). + + - name: weight + type: long + description: > + Total weight (for backends), or server weight (for servers). + + - name: downtime + type: long + description: > + Total downtime (in seconds). For backends, this value is the downtime + for the whole backend, not the sum of the downtime for the servers. + + - name: component_type + type: integer + description: > + Component type (0=frontend, 1=backend, 2=server, or 3=socket/listener). + + - name: process_id + type: integer + description: > + Process ID (0 for first instance, 1 for second, and so on). + + - name: service_name + type: keyword + description: > + Service name (FRONTEND for frontend, BACKEND for backend, or any name for server/listener). + + - name: in.bytes + type: long + format: bytes + description: > + Bytes in. + + - name: out.bytes + type: long + format: bytes + description: > + Bytes out. + + - name: last_change + type: integer + description: > + Number of seconds since the last UP->DOWN or DOWN->UP transition. + + - name: throttle.pct + type: scaled_float + format: percentage + description: > + Current throttle percentage for the server when slowstart + is active, or no value if slowstart is inactive. + + - name: selected.total + type: long + description: > + Total number of times a server was selected, either for new + sessions, or when re-dispatching. For servers, this field reports the + the number of times the server was selected. + + - name: tracked.id + type: long + description: > + ID of the proxy/server if tracking is enabled. + + - name: connection + type: group + fields: + + - name: total + type: long + description: > + Cumulative number of connections. + + - name: retried + type: long + description: > + Number of times a connection to a server was retried. + + - name: time.avg + type: long + description: > + Average connect time in ms over the last 1024 requests. + + - name: request + type: group + fields: + + - name: denied + type: long + description: > + Requests denied because of security concerns. + + * For TCP this is because of a matched tcp-request content rule. + * For HTTP this is because of a matched http-request or tarpit rule. + + - name: queued.current + type: long + description: > + Current queued requests. For backends, this field reports the number + of requests queued without a server assigned. + + - name: queued.max + type: long + description: > + Maximum value of queued.current. + + - name: errors + type: long + description: > + Request errors. Some of the possible causes are: + + * early termination from the client, before the request has been sent + * read error from the client + * client timeout + * client closed connection + * various bad requests from the client. + * request was tarpitted. + + - name: redispatched + type: long + description: > + Number of times a request was redispatched to another server. For + servers, this field reports the number of times the server was + switched away from. + + - name: connection.errors + type: long + description: > + Number of requests that encountered an error trying to + connect to a server. For backends, this field reports the sum of + the stat for all backend servers, plus any connection errors not + associated with a particular server (such as the backend having no + active servers). + + - name: rate + type: group + description: > + fields: + - name: value + type: long + description: > + Number of HTTP requests per second over the last elapsed second. + - name: max + type: long + description: > + Maximum number of HTTP requests per second. + + - name: total + type: long + description: > + Total number of HTTP requests received. + + + - name: response + type: group + fields: + + - name: errors + type: long + description: > + Number of response errors. This value includes the number of data + transfers aborted by the server (haproxy.stat.server.aborted). + Some other errors are: + + * write errors on the client socket (won't be counted for the server stat) + * failure applying filters to the response + + + - name: time.avg + type: long + description: > + Average response time in ms over the last 1024 requests (0 for TCP). + + - name: denied + type: integer + description: > + Responses denied because of security concerns. For HTTP this is + because of a matched http-request rule, or "option checkcache". + + - name: http + type: group + description: > + + fields: + - name: 1xx + type: long + description: > + HTTP responses with 1xx code. + + - name: 2xx + type: long + description: > + HTTP responses with 2xx code. + + - name: 3xx + type: long + description: > + HTTP responses with 3xx code. + + - name: 4xx + type: long + description: > + HTTP responses with 4xx code. + + - name: 5xx + type: long + description: > + HTTP responses with 5xx code. + + - name: other + type: long + description: > + HTTP responses with other codes (protocol error). + + + - name: session + type: group + fields: + + - name: current + type: long + description: > + Number of current sessions. + + - name: max + type: long + description: > + Maximum number of sessions. + + - name: limit + type: long + description: > + Configured session limit. + + - name: rate + type: group + fields: + - name: value + type: integer + description: > + Number of sessions per second over the last elapsed second. + + - name: limit + type: integer + description: > + Configured limit on new sessions per second. + + - name: max + type: integer + description: > + Maximum number of new sessions per second. + + + - name: check + type: group + description: > + + fields: + - name: status + type: keyword + description: > + Status of the last health check. One of: + + UNK -> unknown + INI -> initializing + SOCKERR -> socket error + L4OK -> check passed on layer 4, no upper layers testing enabled + L4TOUT -> layer 1-4 timeout + L4CON -> layer 1-4 connection problem, for example + "Connection refused" (tcp rst) or "No route to host" (icmp) + L6OK -> check passed on layer 6 + L6TOUT -> layer 6 (SSL) timeout + L6RSP -> layer 6 invalid response - protocol error + L7OK -> check passed on layer 7 + L7OKC -> check conditionally passed on layer 7, for example 404 with + disable-on-404 + L7TOUT -> layer 7 (HTTP/SMTP) timeout + L7RSP -> layer 7 invalid response - protocol error + L7STS -> layer 7 response error, for example HTTP 5xx + + - name: code + type: long + description: > + Layer 5-7 code, if available. + + - name: duration + type: long + description: > + Time in ms that it took to finish the last health check. + + - name: health.last + type: keyword + description: > + The result of the last health check. + + - name: health.fail + type: long + description: > + Number of failed checks. + + - name: agent.last + type: integer + description: > + + - name: failed + type: long + description: > + Number of checks that failed while the server was up. + + - name: down + type: long + description: > + Number of UP->DOWN transitions. For backends, this value is the + number of transitions to the whole backend being down, rather than + the sum of the transitions for each server. + + - name: client.aborted + type: integer + description: > + Number of data transfers aborted by the client. + + + - name: server + type: group + description: > + fields: + + - name: id + type: integer + description: > + Server ID (unique inside a proxy). + + - name: aborted + type: integer + description: > + Number of data transfers aborted by the server. This value is + included in haproxy.stat.response.errors. + + - name: active + type: integer + description: > + Number of backend servers that are active, meaning that they are + healthy and can receive requests from the load balancer. + + - name: backup + type: integer + description: > + Number of backend servers that are backup servers. + + + - name: compressor + type: group + description: > + + fields: + - name: in.bytes + type: long + format: bytes + description: > + Number of HTTP response bytes fed to the compressor. + + - name: out.bytes + type: integer + format: bytes + description: > + Number of HTTP response bytes emitted by the compressor. + + - name: bypassed.bytes + type: long + format: bytes + description: > + Number of bytes that bypassed the HTTP compressor (CPU/BW limit). + + - name: response.bytes + type: long + format: bytes + description: > + Number of HTTP responses that were compressed. + + - name: proxy + type: group + description: > + + fields: + - name: id + type: integer + description: > + Unique proxy ID. + + - name: name + type: keyword + description: > + Proxy name. + + + - name: queue + type: group + description: > + + fields: + - name: limit + type: integer + description: > + Configured queue limit (maxqueue) for the server, or nothing if the + value of maxqueue is 0 (meaning no limit). + + - name: time.avg + type: integer + description: > + The average queue time in ms over the last 1024 requests. + + + + + +- key: http + title: "HTTP" + description: > + HTTP module + fields: + - name: http + type: group + description: > + fields: + - name: request + type: group + description: > + HTTP request information + fields: + - name: header + type: object + description: > + The HTTP headers sent + - name: method + type: keyword + description: > + The HTTP method used + - name: body + type: keyword + description: > + The HTTP payload sent + - name: response + type: group + description: > + HTTP response information + fields: + - name: header + type: object + description: > + The HTTP headers received + - name: status_code + type: keyword + description: > + The HTTP status code + - name: body + type: keyword + description: > + The HTTP payload received + - name: json + type: group + description: > + json metricset + fields: + + - name: server + type: group + description: > + server + fields: + +- key: jolokia + title: "Jolokia" + description: > + beta[] + + Jolokia module + short_config: false + fields: + - name: jolokia + type: group + description: > + jolokia contains metrics exposed via jolokia agent + fields: + +- key: kafka + title: "Kafka" + description: > + Kafka module + + beta[] + short_config: false + fields: + - name: kafka + type: group + description: > + fields: + - name: consumergroup + type: group + description: > + consumergroup + fields: + - name: broker + type: group + description: > + Broker Consumer Group Information have been read from (Broker handling + the consumer group). + fields: + - name: id + type: long + description: > + Broker id + + - name: address + type: keyword + description: > + Broker address + + - name: id + type: keyword + description: Consumer Group ID + + - name: topic + type: keyword + description: Topic name + + - name: partition + type: long + description: Partition ID + + - name: offset + type: long + description: consumer offset into partition being read + + - name: meta + type: text + description: custom consumer meta data string + + - name: error.code + type: long + description: > + kafka consumer/partition error code. + + - name: client + type: group + description: > + Assigned client reading events from partition + fields: + - name: id + type: keyword + description: Client ID (kafka setting client.id) + + - name: host + type: keyword + description: Client host + + - name: member_id + type: keyword + description: internal consumer group member ID + + - name: partition + type: group + description: > + partition + fields: + - name: offset + type: group + description: > + Available offsets of the given partition. + fields: + - name: newest + type: long + description: > + Newest offset of the partition. + - name: oldest + type: long + description: > + Oldest offset of the partition. + + - name: partition + type: group + description: > + Partition data. + fields: + - name: id + type: long + description: > + Partition id. + + - name: leader + type: long + description: > + Leader id (broker). + - name: isr + type: array + description: > + List of isr ids. + - name: replica + type: long + description: > + Replica id (broker). + + - name: insync_replica + type: boolean + description: > + Indicates if replica is included in the in-sync replicate set (ISR). + + - name: error.code + type: long + description: > + Error code from fetching partition. + + - name: topic.error.code + type: long + description: > + topic error code. + - name: topic.name + type: keyword + description: > + Topic name + + - name: broker.id + type: long + description: > + Broker id + - name: broker.address + type: keyword + description: > + Broker address + + + +- key: kibana + title: "Kibana" + description: > + experimental[] + + Kibana module + short_config: false + fields: + - name: kibana + type: group + description: > + fields: + - name: status + type: group + description: > + Status fields + fields: + - name: name + type: keyword + description: > + Kibana instance name. + - name: uuid + type: keyword + description: > + Kibana instance uuid. + - name: version.number + type: keyword + description: > + Kibana version number. + - name: status.overall.state + type: keyword + description: > + Kibana overall state. + - name: metrics + type: group + description: > + Metrics fields + fields: + - name: concurrent_connections + type: long + description: > + Current concurrent connections. + - name: requests + type: group + description: > + Request statistics. + fields: + - name: disconnects + type: long + description: > + Total number of disconnected connections. + - name: total + type: long + description: > + Total number of connections. + +- key: kubernetes + title: "Kubernetes" + description: > + beta[] + + Kubernetes metrics + short_config: false + fields: + - name: kubernetes + type: group + description: > + Information and statistics of pods managed by kubernetes. + fields: + - name: container + type: group + description: > + kubernetes container metrics + fields: + - name: start_time + type: date + description: > + Start time + - name: cpu + type: group + description: > + CPU usage metrics + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + description: > + Container CPU Core usage nanoseconds + - name: nanocores + type: long + description: > + CPU used nanocores + - name: logs + type: group + description: > + Logs info + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Logs available capacity in bytes + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Logs total capacity in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Logs used capacity in bytes + - name: inodes + type: group + fields: + - name: count + type: long + description: > + Total available inodes + - name: free + type: long + description: > + Total free inodes + - name: used + type: long + description: > + Total used inodes + - name: memory + type: group + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total available memory + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total memory usage + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Working set memory usage + - name: pagefaults + type: long + description: > + Number of page faults + - name: majorpagefaults + type: long + description: > + Number of major page faults + - name: rootfs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Root filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Root filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Root filesystem total used in bytes + - name: inodes + type: group + fields: + - name: used + type: long + description: > + Used inodes + + - name: event + type: group + description: > + The Kubernetes events metricset collects events that are generated by objects running + inside of Kubernetes + fields: + - name: count + type: long + description: > + Count field records the number of times the particular event has occured + fields: + - name: timestamp + type: group + fields: + - name: first_occurrence + type: date + description: > + Timestamp of first occurrence of event + - name: last_occurrence + type: date + description: > + Timestamp of last occurrence of event + - name: message + type: keyword + description: > + Message recorded for the given event + - name: reason + type: keyword + description: > + Reason recorded for the given event + - name: type + type: keyword + description: > + Type of the given event + - name: metadata + type: group + description: > + Metadata associated with the given event + fields: + - name: timestamp + type: group + fields: + - name: created + type: date + description: > + Timestamp of creation of the given event + - name: name + type: keyword + description: > + Name of the event + - name: namespace + type: keyword + description: > + Namespace in which event was generated + - name: resource_version + type: keyword + description: > + Version of the event resource + - name: uid + type: keyword + description: > + Unique identifier to the event object + - name: self_link + type: keyword + description: > + URL representing the event + - name: involved_object + type: group + description: > + Metadata associated with the given involved object + fields: + - name: api_version + type: keyword + description: > + API version of the object + - name: kind + type: keyword + description: > + API kind of the object + - name: name + type: keyword + description: > + name of the object + - name: resource_version + type: keyword + description: > + resource version of the object + - name: uid + type: keyword + description: > + UUID version of the object + + - name: node + type: group + description: > + kubernetes node metrics + fields: + - name: name + type: keyword + description: > + Node name + - name: start_time + type: date + description: > + Start time + - name: cpu + type: group + description: > + CPU usage metrics + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + description: > + Node CPU Core usage nanoseconds + - name: nanocores + type: long + description: > + CPU used nanocores + - name: memory + type: group + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total available memory + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total memory usage + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Working set memory usage + - name: pagefaults + type: long + description: > + Number of page faults + - name: majorpagefaults + type: long + description: > + Number of major page faults + - name: network + type: group + fields: + - name: rx + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Received bytes + - name: errors + type: long + description: > + Rx errors + - name: tx + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Transmitted bytes + - name: errors + type: long + description: > + Tx errors + - name: fs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total used in bytes + - name: inodes + type: group + fields: + - name: used + type: long + description: > + Number of used inodes + - name: count + type: long + description: > + Number of inodes + - name: free + type: long + description: > + Number of free inodes + - name: runtime + type: group + fields: + - name: imagefs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Image filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Image filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Image filesystem total used in bytes + + - name: pod + type: group + description: > + kubernetes pod metrics + fields: + - name: start_time + type: date + description: > + Start time + - name: network + type: group + fields: + - name: rx + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Received bytes + - name: errors + type: long + description: > + Rx errors + - name: tx + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Transmitted bytes + - name: errors + type: long + description: > + Tx errors + + - name: container + type: group + description: > + kubernetes container metrics + fields: + - name: id + type: keyword + description: + Container id + - name: status + type: group + fields: + - name: phase + type: keyword + description: > + Container phase (running, waiting, terminated) + - name: ready + type: boolean + description: > + Container ready status + - name: restarts + type: integer + description: > + Container restarts count + - name: cpu + type: group + fields: + - name: limit.nanocores + type: long + description: > + Container CPU nanocores limit + - name: request.nanocores + type: long + description: > + Container CPU requested nanocores + - name: memory + type: group + fields: + - name: limit.bytes + type: long + format: bytes + description: > + Container memory limit in bytes + - name: request.bytes + type: long + format: bytes + description: > + Container requested memory in bytes + + - name: deployment + type: group + description: > + kubernetes deployment metrics + fields: + - name: name + type: keyword + description: > + Kubernetes deployment name + - name: paused + type: boolean + description: > + Kubernetes deployment paused status + - name: replicas + type: group + description: > + Kubernetes deployment replicas info + fields: + - name: desired + type: integer + description: > + Deployment number of desired replicas (spec) + - name: available + type: integer + description: > + Deployment available replicas + - name: unavailable + type: integer + description: > + Deployment unavailable replicas + - name: updated + type: integer + description: > + Deployment updated replicas + + - name: node + type: group + description: > + kubernetes node metrics + fields: + - name: status + type: group + fields: + - name: ready + type: keyword + description: > + Node ready status (true, false or unknown) + - name: unschedulable + type: boolean + description: > + Node unschedulable status + - name: cpu + type: group + fields: + - name: allocatable.cores + type: long + description: > + Node CPU allocatable cores + - name: capacity.cores + type: long + description: > + Node CPU capacity cores + - name: memory + type: group + fields: + - name: allocatable.bytes + type: long + format: bytes + description: > + Node allocatable memory in bytes + - name: capacity.bytes + type: long + format: bytes + description: > + Node memory capacity in bytes + - name: pod + type: group + fields: + - name: allocatable.total + type: long + description: > + Node allocatable pods + - name: capacity.total + type: long + description: > + Node pod capacity + + - name: pod + type: group + description: > + kubernetes pod metrics + fields: + - name: ip + type: ip + description: > + Kubernetes pod IP + - name: host_ip + type: ip + description: > + Kubernetes pod host IP + - name: status + type: group + description: > + Kubernetes pod status metrics + fields: + - name: phase + type: keyword + description: > + Kubernetes pod phase (Running, Pending...) + - name: ready + type: keyword + description: > + Kubernetes pod ready status (true, false or unknown) + - name: scheduled + type: keyword + description: > + Kubernetes pod scheduled status (true, false, unknown) + + - name: replicaset + type: group + description: > + kubernetes replica set metrics + fields: + - name: name + type: keyword + description: > + Kubernetes replica set name + - name: replicas + type: group + description: > + Kubernetes replica set paused status + fields: + - name: available + type: long + description: > + The number of replicas per ReplicaSet + - name: desired + type: long + description: > + The number of replicas per ReplicaSet + - name: ready + type: long + description: > + The number of ready replicas per ReplicaSet + - name: observed + type: long + description: > + The generation observed by the ReplicaSet controller + - name: labeled + type: long + description: > + The number of fully labeled replicas per ReplicaSet + + - name: system + type: group + description: > + kubernetes system containers metrics + fields: + - name: container + type: keyword + description: > + Container name + - name: start_time + type: date + description: > + Start time + - name: cpu + type: group + description: > + CPU usage metrics + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + description: > + CPU Core usage nanoseconds + - name: nanocores + type: long + description: > + CPU used nanocores + - name: memory + type: group + fields: + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total memory usage + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Working set memory usage + - name: pagefaults + type: long + description: > + Number of page faults + - name: majorpagefaults + type: long + description: > + Number of major page faults + + - name: volume + type: group + description: > + kubernetes volume metrics + fields: + - name: name + type: keyword + description: > + Volume name + - name: fs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Filesystem total used in bytes + - name: inodes + type: group + fields: + - name: used + type: long + description: > + Used inodes + - name: free + type: long + description: > + Free inodes + - name: count + type: long + description: > + Total inodes + +- key: memcached + title: "Memcached" + description: > + beta[] + + Memcached module + short_config: false + fields: + - name: memcached + type: group + description: > + fields: + - name: stats + type: group + description: > + stats + fields: + - name: pid + type: long + description: > + Current process ID of the Memcached task. + + - name: uptime.sec + type: long + description: > + Memcached server uptime. + + - name: threads + type: long + description: > + Number of threads used by the current Memcached server process. + + - name: connections.current + type: long + description: > + Number of open connections to this Memcached server, should be the same + value on all servers during normal operation. + + - name: connections.total + type: long + description: > + Numer of successful connect attempts to this server since it has been started. + + - name: get.hits + type: long + description: > + Number of successful "get" commands (cache hits) since startup, divide them + by the "cmd_get" value to get the cache hitrate. + + - name: get.misses + type: long + description: > + Number of failed "get" requests because nothing was cached for this key + or the cached value was too old. + + - name: cmd.get + type: long + description: > + Number of "get" commands received since server startup not counting if they + were successful or not. + + - name: cmd.set + type: long + description: > + Number of "set" commands serviced since startup. + + - name: read.bytes + type: long + formate: bytes + description: > + Total number of bytes received from the network by this server. + + - name: written.bytes + type: long + formate: bytes + description: > + Total number of bytes send to the network by this server. + + - name: items.current + type: long + description: > + Number of items currently in this server's cache. + + - name: items.total + type: long + formate: bytes + description: > + Number of items stored ever stored on this server. This is no "maximum item + count" value but a counted increased by every new item stored in the cache. + + - name: evictions + type: long + formate: bytes + description: > + Number of objects removed from the cache to free up memory for new items + because Memcached reached it's maximum memory setting (limit_maxbytes). + +- key: mongodb + title: "MongoDB" + description: > + Metrics collected from MongoDB servers. + short_config: false + fields: + - name: mongodb + type: group + description: > + MongoDB metrics. + fields: + - name: dbstats + type: group + description: > + dbstats provides an overview of a particular mongo database. This document + is most concerned with data volumes of a database. + fields: + - name: avg_obj_size.bytes + type: long + format: bytes + + - name: collections + type: integer + + - name: data_size.bytes + type: long + format: bytes + + - name: db + type: keyword + + - name: file_size.bytes + type: long + format: bytes + + - name: index_size.bytes + type: long + format: bytes + + - name: indexes + type: long + + - name: num_extents + type: long + + - name: objects + type: long + + - name: storage_size.bytes + type: long + format: bytes + + - name: ns_size_mb.mb + type: long + + - name: data_file_version + type: group + fields: + - name: major + type: long + + - name: minor + type: long + + - name: extent_free_list + type: group + fields: + - name: num + type: long + + - name: size.bytes + type: long + format: bytes + + - name: status + type: group + description: > + MongoDB server status metrics. + fields: + - name: version + type: keyword + description: > + Instance version. + - name: uptime.ms + type: long + description: > + Instance uptime in milliseconds. + - name: local_time + type: date + description: > + Local time as reported by the MongoDB instance. + + - name: asserts.regular + type: long + description: > + Number of regular assertions produced by the server. + - name: asserts.warning + type: long + description: > + Number of warning assertions produced by the server. + - name: asserts.msg + type: long + description: > + Number of msg assertions produced by the server. + - name: asserts.user + type: long + description: > + Number of user assertions produced by the server. + - name: asserts.rollovers + type: long + description: > + Number of rollovers assertions produced by the server. + + - name: background_flushing + type: group + description: > + Data about the process MongoDB uses to write data to disk. This data is + only available for instances that use the MMAPv1 storage engine. + fields: + - name: flushes + type: long + description: > + A counter that collects the number of times the database has + flushed all writes to disk. + - name: total.ms + type: long + description: > + The total number of milliseconds (ms) that the mongod processes have + spent writing (i.e. flushing) data to disk. Because this is an + absolute value, consider the value of `flushes` and `average_ms` to + provide better context for this datum. + - name: average.ms + type: long + description: > + The average time spent flushing to disk per flush event. + - name: last.ms + type: long + description: > + The amount of time, in milliseconds, that the last flush operation + took to complete. + - name: last_finished + type: date + description: > + A timestamp of the last completed flush operation. + + - name: connections + type: group + description: > + Data regarding the current status of incoming connections and + availability of the database server. + fields: + - name: current + type: long + description: > + The number of connections to the database server from clients. This + number includes the current shell session. Consider the value of + `available` to add more context to this datum. + - name: available + type: long + description: > + The number of unused available incoming connections the database + can provide. + - name: total_created + type: long + description: > + A count of all incoming connections created to the server. This + number includes connections that have since closed. + + - name: journaling + type: group + description: > + Data about the journaling-related operations and performance. Journaling + information only appears for mongod instances that use the MMAPv1 + storage engine and have journaling enabled. + fields: + - name: commits + type: long + description: > + The number of transactions written to the journal during the last + journal group commit interval. + - name: journaled.mb + type: long + description: > + The amount of data in megabytes (MB) written to journal during the + last journal group commit interval. + - name: write_to_data_files.mb + type: long + description: > + The amount of data in megabytes (MB) written from journal to the + data files during the last journal group commit interval. + - name: compression + type: long + description: > + The compression ratio of the data written to the journal. + - name: commits_in_write_lock + type: long + description: > + Count of the commits that occurred while a write lock was held. + Commits in a write lock indicate a MongoDB node under a heavy write + load and call for further diagnosis. + - name: early_commits + type: long + description: > + The number of times MongoDB requested a commit before the scheduled + journal group commit interval. + - name: times + type: group + description: > + Information about the performance of the mongod instance during the + various phases of journaling in the last journal group commit + interval. + fields: + - name: dt.ms + type: long + description: > + The amount of time over which MongoDB collected the times data. + Use this field to provide context to the other times field values. + - name: prep_log_buffer.ms + type: long + description: > + The amount of time spent preparing to write to the journal. + Smaller values indicate better journal performance. + - name: write_to_journal.ms + type: long + description: > + The amount of time spent actually writing to the journal. File + system speeds and device interfaces can affect performance. + - name: write_to_data_files.ms + type: long + description: > + The amount of time spent writing to data files after journaling. + File system speeds and device interfaces can affect performance. + - name: remap_private_view.ms + type: long + description: > + The amount of time spent remapping copy-on-write memory mapped + views. Smaller values indicate better journal performance. + - name: commits.ms + type: long + description: > + The amount of time spent for commits. + - name: commits_in_write_lock.ms + type: long + description: > + The amount of time spent for commits that occurred while a write + lock was held. + + - name: extra_info + type: group + description: > + Platform specific data. + fields: + - name: heap_usage.bytes + type: long + format: bytes + description: > + The total size in bytes of heap space used by the database process. + Only available on Unix/Linux. + - name: page_faults + type: long + description: > + The total number of page faults that require disk operations. Page + faults refer to operations that require the database server to + access data that isn't available in active memory. + + - name: network + type: group + description: > + Platform specific data. + fields: + - name: in.bytes + type: long + format: bytes + description: > + The amount of network traffic, in bytes, received by this database. + - name: out.bytes + type: long + format: bytes + description: > + The amount of network traffic, in bytes, sent from this database. + - name: requests + type: long + description: > + The total number of requests received by the server. + + - name: opcounters + type: group + description: > + An overview of database operations by type. + fields: + - name: insert + type: long + description: > + The total number of insert operations received since the mongod + instance last started. + - name: query + type: long + description: > + The total number of queries received since the mongod instance last + started. + - name: update + type: long + description: > + The total number of update operations received since the mongod + instance last started. + - name: delete + type: long + description: > + The total number of delete operations received since the mongod + instance last started. + - name: getmore + type: long + description: > + The total number of getmore operations received since the mongod + instance last started. + - name: command + type: long + description: > + The total number of commands issued to the database since the mongod + instance last started. + + - name: opcounters_replicated + type: group + description: > + An overview of database replication operations by type. + fields: + - name: insert + type: long + description: > + The total number of replicated insert operations received since the + mongod instance last started. + - name: query + type: long + description: > + The total number of replicated queries received since the mongod + instance last started. + - name: update + type: long + description: > + The total number of replicated update operations received since the + mongod instance last started. + - name: delete + type: long + description: > + The total number of replicated delete operations received since the + mongod instance last started. + - name: getmore + type: long + description: > + The total number of replicated getmore operations received since the + mongod instance last started. + - name: command + type: long + description: > + The total number of replicated commands issued to the database since + the mongod instance last started. + + - name: memory + type: group + description: > + Data about the current memory usage of the mongod server. + fields: + - name: bits + type: long + description: > + Either 64 or 32, depending on which target architecture was specified + during the mongod compilation process. + - name: resident.mb + type: long + description: > + The amount of RAM, in megabytes (MB), currently used by the database + process. + - name: virtual.mb + type: long + description: > + The amount, in megabytes (MB), of virtual memory used by the mongod + process. + - name: mapped.mb + type: long + description: > + The amount of mapped memory, in megabytes (MB), used by the database. + Because MongoDB uses memory-mapped files, this value is likely to be + to be roughly equivalent to the total size of your database or + databases. + - name: mapped_with_journal.mb + type: long + description: > + The amount of mapped memory, in megabytes (MB), including the memory + used for journaling. + - name: write_backs_queued + type: boolean + description: > + True when there are operations from a mongos instance queued for retrying. + - name: storage_engine.name + type: keyword + description: > + A string that represents the name of the current storage engine. + + - name: wired_tiger + type: group + description: > + Statistics about the WiredTiger storage engine. + fields: + - name: concurrent_transactions + type: group + description: > + Statistics about the transactions currently in progress. + fields: + - name: write.out + type: long + description: > + Number of concurrent write transaction in progress. + - name: write.available + type: long + description: > + Number of concurrent write tickets available. + - name: write.total_tickets + type: long + description: > + Number of total write tickets. + - name: read.out + type: long + description: > + Number of concurrent read transaction in progress. + - name: read.available + type: long + description: > + Number of concurrent read tickets available. + - name: read.total_tickets + type: long + description: > + Number of total read tickets. + - name: cache + type: group + description: > + Statistics about the cache and page evictions from the cache. + fields: + - name: maximum.bytes + type: long + format: bytes + description: > + Maximum cache size. + - name: used.bytes + type: long + format: bytes + description: > + Size in byte of the data currently in cache. + - name: dirty.bytes + type: long + format: bytes + description: > + Size in bytes of the dirty data in the cache. + - name: pages.read + type: long + description: > + Number of pages read into the cache. + - name: pages.write + type: long + description: > + Number of pages written from the cache. + - name: pages.evicted + type: long + description: > + Number of pages evicted from the cache. + - name: log + type: group + description: > + Statistics about the write ahead log used by WiredTiger. + fields: + - name: size.bytes + type: long + format: bytes + description: > + Total log size in bytes. + - name: write.bytes + type: long + format: bytes + description: > + Number of bytes written into the log. + - name: max_file_size.bytes + type: long + format: bytes + description: > + Maximum file size. + - name: flushes + type: long + description: > + Number of flush operations. + - name: writes + type: long + description: > + Number of write operations. + - name: scans + type: long + description: > + Number of scan operations. + - name: syncs + type: long + description: > + Number of sync operations. + +- key: mysql + title: "MySQL" + description: > + MySQL server status metrics collected from MySQL. + short_config: false + fields: + - name: mysql + type: group + description: > + `mysql` contains the metrics that were obtained from MySQL + query. + fields: + - name: status + type: group + description: > + `status` contains the metrics that were obtained by the status SQL query. + fields: + - name: aborted + type: group + description: > + Aborted status fields. + fields: + - name: clients + type: long + description: > + The number of connections that were aborted because the client died without closing the connection properly. + + - name: connects + type: long + description: > + The number of failed attempts to connect to the MySQL server. + + - name: binlog + type: group + description: > + fields: + - name: cache.disk_use + type: long + description: > + + - name: cache.use + type: long + description: > + + - name: bytes + type: group + description: > + Bytes stats. + fields: + - name: received + format: bytes + type: long + description: > + The number of bytes received from all clients. + + - name: sent + type: long + format: bytes + description: > + The number of bytes sent to all clients. + + - name: threads + type: group + description: > + Threads stats. + fields: + - name: cached + type: long + description: > + The number of cached threads. + + - name: created + type: long + description: > + The number of created threads. + + - name: connected + type: long + description: > + The number of connected threads. + + - name: running + type: long + description: > + The number of running threads. + + - name: connections + type: long + description: > + + - name: created + type: group + description: > + fields: + - name: tmp.disk_tables + type: long + description: > + + - name: tmp.files + type: long + description: > + + - name: tmp.tables + type: long + description: > + + - name: delayed + type: group + description: > + fields: + - name: errors + type: long + description: > + + - name: insert_threads + type: long + description: > + + - name: writes + type: long + description: > + + - name: flush_commands + type: long + description: > + + - name: max_used_connections + type: long + description: > + + - name: open + type: group + description: > + fields: + - name: files + type: long + description: > + + - name: streams + type: long + description: > + + - name: tables + type: long + description: > + + - name: opened_tables + type: long + description: > + + - name: command + type: group + description: > + fields: + - name: delete + type: long + description: > + The number of DELETE queries since startup. + + - name: insert + type: long + description: > + The number of INSERT queries since startup. + + - name: select + type: long + description: > + The number of SELECT queries since startup. + + - name: update + type: long + description: > + The number of UPDATE queries since startup. + +- key: nginx + title: "Nginx" + description: > + Nginx server status metrics collected from various modules. + short_config: false + fields: + - name: nginx + type: group + description: > + `nginx` contains the metrics that were scraped from nginx. + fields: + - name: stubstatus + type: group + description: > + `stubstatus` contains the metrics that were scraped from the ngx_http_stub_status_module status page. + fields: + - name: hostname + type: keyword + description: > + Nginx hostname. + - name: active + type: long + description: > + The current number of active client connections including Waiting connections. + - name: accepts + type: long + description: > + The total number of accepted client connections. + - name: handled + type: long + description: > + The total number of handled client connections. + - name: dropped + type: long + description: > + The total number of dropped client connections. + - name: requests + type: long + description: > + The total number of client requests. + - name: current + type: long + description: > + The current number of client requests. + - name: reading + type: long + description: > + The current number of connections where Nginx is reading the request header. + - name: writing + type: long + description: > + The current number of connections where Nginx is writing the response back to the client. + - name: waiting + type: long + description: > + The current number of idle client connections waiting for a request. + +- key: php_fpm + title: "PHP_FPM" + description: > + beta[] + + PHP-FPM server status metrics collected from PHP-FPM. + short_config: false + fields: + - name: php_fpm + type: group + description: > + `php_fpm` contains the metrics that were obtained from PHP-FPM status + page call. + fields: + - name: pool + type: group + description: > + `pool` contains the metrics that were obtained from the PHP-FPM process + pool. + fields: + - name: name + type: keyword + description: > + The name of the pool. + - name: connections + type: group + description: > + Connection state specific statistics. + fields: + - name: accepted + type: long + description: > + The number of incoming requests that the PHP-FPM server has accepted; + when a connection is accepted it is removed from the listen queue. + - name: queued + type: long + description: > + The current number of connections that have been initiated, but not + yet accepted. If this value is non-zero it typically means that all + the available server processes are currently busy, and there are no + processes available to serve the next request. Raising + `pm.max_children` (provided the server can handle it) should help + keep this number low. This property follows from the fact that + PHP-FPM listens via a socket (TCP or file based), and thus inherits + some of the characteristics of sockets. + - name: processes + type: group + description: > + Process state specific statistics. + fields: + - name: idle + type: long + description: > + The number of servers in the `waiting to process` state (i.e. not + currently serving a page). This value should fall between the + `pm.min_spare_servers` and `pm.max_spare_servers` values when the + process manager is `dynamic`. + - name: active + type: long + description: > + The number of servers current processing a page - the minimum is `1` + (so even on a fully idle server, the result will be not read `0`). + - name: slow_requests + type: long + description: > + The number of times a request execution time has exceeded + `request_slowlog_timeout`. + +- key: postgresql + title: "PostgreSQL" + description: > + Metrics collected from PostgreSQL servers. + short_config: false + fields: + - name: postgresql + type: group + description: > + PostgreSQL metrics. + fields: + - name: activity + type: group + description: > + One document per server process, showing information related to the current + activity of that process, such as state and current query. Collected by + querying pg_stat_activity. + fields: + - name: database.oid + type: long + description: > + OID of the database this backend is connected to. + - name: database.name + type: keyword + description: > + Name of the database this backend is connected to. + - name: pid + type: long + description: > + Process ID of this backend. + - name: user.id + type: long + description: > + OID of the user logged into this backend. + - name: user.name + description: > + Name of the user logged into this backend. + - name: application_name + description: > + Name of the application that is connected to this backend. + - name: client.address + description: > + IP address of the client connected to this backend. + - name: client.hostname + description: > + Host name of the connected client, as reported by a reverse DNS lookup of client_addr. + - name: client.port + type: long + description: > + TCP port number that the client is using for communication with this + backend, or -1 if a Unix socket is used. + - name: backend_start + type: date + description: > + Time when this process was started, i.e., when the client connected to + the server. + - name: transaction_start + type: date + description: > + Time when this process' current transaction was started. + - name: query_start + type: date + description: > + Time when the currently active query was started, or if state is not + active, when the last query was started. + - name: state_change + type: date + description: > + Time when the state was last changed. + - name: waiting + type: boolean + description: > + True if this backend is currently waiting on a lock. + - name: state + description: > + Current overall state of this backend. Possible values are: + + * active: The backend is executing a query. + * idle: The backend is waiting for a new client command. + * idle in transaction: The backend is in a transaction, but is not + currently executing a query. + * idle in transaction (aborted): This state is similar to idle in + transaction, except one of the statements in the transaction caused + an error. + * fastpath function call: The backend is executing a fast-path function. + * disabled: This state is reported if track_activities is disabled in this backend. + - name: query + description: > + Text of this backend's most recent query. If state is active this field + shows the currently executing query. In all other states, it shows the + last query that was executed. + + + - name: bgwriter + type: group + description: > + Statistics about the background writer process's activity. Collected using the + pg_stat_bgwriter query. + fields: + - name: checkpoints.scheduled + type: long + description: > + Number of scheduled checkpoints that have been performed. + - name: checkpoints.requested + type: long + description: > + Number of requested checkpoints that have been performed. + - name: checkpoints.times.write.ms + type: float + description: > + Total amount of time that has been spent in the portion of checkpoint + processing where files are written to disk, in milliseconds. + - name: checkpoints.times.sync.ms + type: float + description: > + Total amount of time that has been spent in the portion of checkpoint + processing where files are synchronized to disk, in milliseconds. + - name: buffers.checkpoints + type: long + description: > + Number of buffers written during checkpoints. + - name: buffers.clean + type: long + description: > + Number of buffers written by the background writer. + - name: buffers.clean_full + type: long + description: > + Number of times the background writer stopped a cleaning scan because it + had written too many buffers. + - name: buffers.backend + type: long + description: > + Number of buffers written directly by a backend. + - name: buffers.backend_fsync + type: long + description: > + Number of times a backend had to execute its own fsync call (normally + the background writer handles those even when the backend does its own + write) + - name: buffers.allocated + type: long + description: > + Number of buffers allocated. + - name: stats_reset + type: date + description: > + Time at which these statistics were last reset. + + - name: database + type: group + description: > + One row per database, showing database-wide statistics. Collected by querying + pg_stat_database + fields: + - name: oid + type: long + description: > + OID of the database this backend is connected to. + - name: name + type: keyword + description: > + Name of the database this backend is connected to. + - name: number_of_backends + type: long + description: > + Number of backends currently connected to this database. + - name: transactions.commit + type: long + description: > + Number of transactions in this database that have been committed. + - name: transactions.rollback + type: long + description: > + Number of transactions in this database that have been rolled back. + - name: blocks.read + type: long + description: > + Number of disk blocks read in this database. + - name: blocks.hit + type: long + description: > + Number of times disk blocks were found already in the buffer cache, so + that a read was not necessary (this only includes hits in the PostgreSQL + buffer cache, not the operating system's file system cache). + - name: blocks.time.read.ms + type: long + description: > + Time spent reading data file blocks by backends in this database, in + milliseconds. + - name: blocks.time.write.ms + type: long + description: > + Time spent writing data file blocks by backends in this database, in + milliseconds. + - name: rows.returned + type: long + description: > + Number of rows returned by queries in this database. + - name: rows.fetched + type: long + description: > + Number of rows fetched by queries in this database. + - name: rows.inserted + type: long + description: > + Number of rows inserted by queries in this database. + - name: rows.updated + type: long + description: > + Number of rows updated by queries in this database. + - name: rows.deleted + type: long + description: > + Number of rows deleted by queries in this database. + - name: conflicts + type: long + description: > + Number of queries canceled due to conflicts with recovery in this + database. + - name: temporary.files + type: long + description: > + Number of temporary files created by queries in this database. All + temporary files are counted, regardless of why the temporary file was + created (e.g., sorting or hashing), and regardless of the log_temp_files + setting. + - name: temporary.bytes + type: long + description: > + Total amount of data written to temporary files by queries in this + database. All temporary files are counted, regardless of why the + temporary file was created, and regardless of the log_temp_files + setting. + - name: deadlocks + type: long + description: > + Number of deadlocks detected in this database. + - name: stats_reset + type: date + description: > + Time at which these statistics were last reset. + + +- key: prometheus + title: "Prometheus" + description: > + beta[] + + Stats collected from Prometheus. + short_config: false + fields: + - name: prometheus + type: group + description: > + fields: + + - name: stats + type: group + description: > + Stats about the Prometheus server. + fields: + - name: notifications + type: group + description: > + Notification stats. + fields: + - name: queue_length + type: long + description: > + Current queue length. + - name: dropped + type: long + description: > + Number of dropped queue events. + - name: processes.open_fds + type: long + description: > + Number of open file descriptors. + - name: storage.chunks_to_persist + type: long + description: > + Number of memory chunks that are not yet persisted to disk. + +- key: rabbitmq + title: "RabbitMQ" + description: > + experimental[] + + RabbitMQ module + fields: + - name: rabbitmq + type: group + description: > + fields: + - name: node + type: group + description: > + node + fields: + - name: disk.free.bytes + type: long + description: > + Disk free space in bytes. + format: bytes + - name: disk.free.limit.bytes + type: long + description: > + Point at which the disk alarm will go off. + format: bytes + - name: fd.total + type: long + description: > + File descriptors available. + - name: fd.used + type: long + description: > + Used file descriptors. + - name: gc.num.count + type: long + description: > + Number of GC operations. + - name: gc.reclaimed.bytes + type: long + description: > + GC bytes reclaimed. + format: bytes + - name: io.file_handle.open_attempt.avg.ms + type: long + description: > + File handle open avg time + - name: io.file_handle.open_attempt.count + type: long + description: > + File handle open attempts + - name: io.read.avg.ms + type: long + description: > + File handle read avg time + - name: io.read.bytes + type: long + description: > + Data read in bytes + format: bytes + - name: io.read.count + type: long + description: > + Data read operations + - name: io.reopen.count + type: long + description: > + Data reopen operations + - name: io.seek.avg.ms + type: long + description: > + Data seek avg time + - name: io.seek.count + type: long + description: > + Data seek operations + - name: io.sync.avg.ms + type: long + description: > + Data sync avg time + - name: io.sync.count + type: long + description: > + Data sync operations + - name: io.write.avg.ms + type: long + description: > + Data write avg time + - name: io.write.bytes + type: long + description: > + Data write in bytes + format: bytes + - name: io.write.count + type: long + description: > + Data write operations + - name: mem.limit.bytes + type: long + description: > + Point at which the memory alarm will go off. + format: bytes + - name: mem.used.bytes + type: long + description: > + Memory used in bytes. + - name: mnesia.disk.tx.count + type: long + description: > + Number of Mnesia transactions which have been performed that required writes to disk. + - name: mnesia.ram.tx.count + type: long + description: > + Number of Mnesia transactions which have been performed that did not require writes to disk. + - name: msg.store_read.count + type: long + description: > + Number of messages which have been read from the message store. + - name: msg.store_write.count + type: long + description: > + Number of messages which have been written to the message store. + - name: name + type: keyword + description: > + Node name + - name: proc.total + type: long + description: > + Maximum number of Erlang processes. + - name: proc.used + type: long + description: > + Number of Erlang processes in use. + - name: processors + type: long + description: > + Number of cores detected and usable by Erlang. + - name: queue.index.journal_write.count + type: long + description: > + Number of records written to the queue index journal. + - name: queue.index.read.count + type: long + description: > + Number of records read from the queue index. + - name: queue.index.write.count + type: long + description: > + Number of records written to the queue index. + - name: run.queue + type: long + description: > + Average number of Erlang processes waiting to run. + - name: socket.total + type: long + description: > + File descriptors available for use as sockets. + - name: socket.used + type: long + description: > + File descriptors used as sockets. + - name: type + type: keyword + description: > + Node type. + - name: uptime + type: long + description: > + Node uptime. + + - name: queue + type: group + description: > + queue + fields: + - name: name + type: keyword + description: > + The name of the queue with non-ASCII characters escaped as in C. + - name: vhost + type: keyword + description: > + Virtual host name with non-ASCII characters escaped as in C. + - name: durable + type: boolean + description: > + Whether or not the queue survives server restarts. + - name: auto_delete + type: boolean + description: > + Whether the queue will be deleted automatically when no longer used. + - name: exclusive + type: boolean + description: > + Whether the queue is exclusive (i.e. has owner_pid). + - name: node + type: keyword + description: > + Node name. + - name: state + type: keyword + description: > + The state of the queue. Normally 'running', but may be "{syncing, MsgCount}" if the queue is synchronising. Queues which are located on cluster nodes that are currently down will be shown with a status of 'down'. + - name: arguments.max_priority + type: long + description: > + Maximum number of priority levels for the queue to support. + - name: consumers.count + type: long + description: > + Number of consumers. + - name: consumers.utilisation.pct + type: long + format: percentage + description: > + Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count. + - name: messages.total.count + type: long + description: > + Sum of ready and unacknowledged messages (queue depth). + - name: messages.ready.count + type: long + description: > + Number of messages ready to be delivered to clients. + - name: messages.unacknowledged.count + type: long + description: > + Number of messages delivered to clients but not yet acknowledged. + - name: messages.persistent.count + type: long + description: > + Total number of persistent messages in the queue (will always be 0 for transient queues). + - name: memory.bytes + type: long + format: bytes + description: > + Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures. + - name: disk.reads.count + type: long + description: > + Total number of times messages have been read from disk by this queue since it started. + - name: disk.writes.count + type: long + description: > + Total number of times messages have been written to disk by this queue since it started. + +- key: redis + title: "Redis" + description: > + Redis metrics collected from Redis. + fields: + - name: redis + type: group + description: > + `redis` contains the information and statistics from Redis. + fields: + - name: info + type: group + description: > + `info` contains the information and statistics returned by the `INFO` command. + fields: + - name: clients + type: group + description: > + Redis client stats. + fields: + - name: connected + type: long + description: > + Number of client connections (excluding connections from slaves). + + - name: longest_output_list + type: long + description: > + Longest output list among current client connections. + + - name: biggest_input_buf + type: long + description: > + Biggest input buffer among current client connections. + + - name: blocked + type: long + description: > + Number of clients pending on a blocking call (BLPOP, BRPOP, BRPOPLPUSH). + + - name: cluster + type: group + description: > + Redis cluster information. + fields: + - name: enabled + type: boolean + description: > + Indicates that the Redis cluster is enabled. + + - name: cpu + type: group + description: > + Redis CPU stats + fields: + - name: used.sys + type: scaled_float + description: > + System CPU consumed by the Redis server. + + - name: used.sys_children + type: scaled_float + description: > + User CPU consumed by the Redis server. + + - name: used.user + type: scaled_float + description: > + System CPU consumed by the background processes. + + - name: used.user_children + type: scaled_float + description: > + User CPU consumed by the background processes. + + - name: memory + type: group + description: > + Redis memory stats. + fields: + - name: used.value + type: long + description: > + format: bytes + Used memory. + + - name: used.rss + type: long + format: bytes + description: > + Used memory rss. + + - name: used.peak + type: long + format: bytes + description: > + Used memory peak. + + - name: used.lua + type: long + format: bytes + description: > + Used memory lua. + + - name: allocator + type: keyword + description: > + Memory allocator. + + - name: persistence + type: group + description: > + Redis CPU stats. + fields: + - name: loading + type: boolean + description: + + - name: rdb + type: group + description: + fields: + - name: last_save.changes_since + type: long + description: + + - name: bgsave.in_progress + type: boolean + description: + + - name: last_save.time + type: long + description: + + - name: bgsave.last_status + type: keyword + description: + + - name: bgsave.last_time.sec + type: long + description: + + - name: bgsave.current_time.sec + type: long + description: + + - name: aof + type: group + description: + fields: + - name: enabled + type: boolean + description: + + - name: rewrite.in_progress + type: boolean + description: + + - name: rewrite.scheduled + type: boolean + description: + + - name: rewrite.last_time.sec + type: long + description: + + - name: rewrite.current_time.sec + type: long + description: + + - name: bgrewrite.last_status + type: keyword + description: + + - name: write.last_status + type: keyword + description: + + - name: replication + type: group + description: > + Replication + fields: + - name: role + type: keyword + description: + + - name: connected_slaves + type: long + description: + + - name: master_offset + type: long + description: + + - name: backlog.active + type: long + description: + + - name: backlog.size + type: long + description: + + - name: backlog.first_byte_offset + type: long + description: + + - name: backlog.histlen + type: long + description: + + - name: server + type: group + description: > + Server info + fields: + - name: version + type: keyword + description: + + - name: git_sha1 + type: keyword + description: + + - name: git_dirty + type: keyword + description: + + - name: build_id + type: keyword + description: + + - name: mode + type: keyword + description: + + - name: os + type: keyword + description: + + - name: arch_bits + type: keyword + description: + + - name: multiplexing_api + type: keyword + description: + + - name: gcc_version + type: keyword + description: + + - name: process_id + type: long + description: + + - name: run_id + type: keyword + description: + + - name: tcp_port + type: long + description: + + - name: uptime + type: long + description: + + - name: hz + type: long + description: + + - name: lru_clock + type: long + description: + + - name: config_file + type: keyword + description: + + - name: stats + type: group + description: > + Redis stats. + fields: + - name: connections.received + type: long + description: + Total number of connections received. + + - name: connections.rejected + type: long + description: + Total number of connections rejected. + + - name: commands_processed + type: long + description: + Total number of commands preocessed. + + - name: net.input.bytes + type: long + description: + Total network input in bytes. + + - name: net.output.bytes + type: long + description: + Total network output in bytes. + + - name: instantaneous.ops_per_sec + type: long + description: + + - name: instantaneous.input_kbps + type: scaled_float + description: + + - name: instantaneous.output_kbps + type: scaled_float + description: + + - name: sync.full + type: long + description: + + - name: sync.partial.ok + type: long + description: + + - name: sync.partial.err + type: long + description: + + - name: keys.expired + type: long + description: + + - name: keys.evicted + type: long + description: + + - name: keyspace.hits + type: long + description: + + - name: keyspace.misses + type: long + description: + + - name: pubsub.channels + type: long + description: + + - name: pubsub.patterns + type: long + description: + + - name: latest_fork_usec + type: long + description: + + - name: migrate_cached_sockets + type: long + description: + + + - name: keyspace + type: group + description: > + `keyspace` contains the information about the keyspaces returned by the `INFO` command. + fields: + - name: id + type: keyword + description: > + Keyspace identifier. + + - name: avg_ttl + type: long + description: > + Average ttl. + + - name: keys + type: long + description: > + Number of keys in the keyspace. + + - name: expires + type: long + description: > + +- key: system + title: "System" + description: > + System status metrics, like CPU and memory usage, that are collected from the operating system. + short_config: true + fields: + - name: system + type: group + description: > + `system` contains local system metrics. + fields: + - name: core + type: group + description: > + `system-core` contains CPU metrics for a single core of a multi-core system. + fields: + - name: id + type: long + description: > + CPU Core number. + + # Percentages + - name: user.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. + + - name: user.ticks + type: long + description: > + The amount of CPU time spent in user space. + + - name: system.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: system.ticks + type: long + description: > + The amount of CPU time spent in kernel space. + + - name: nice.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: nice.ticks + type: long + description: > + The amount of CPU time spent on low-priority processes. + + - name: idle.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: idle.ticks + type: long + description: > + The amount of CPU time spent idle. + + - name: iowait.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: iowait.ticks + type: long + description: > + The amount of CPU time spent in wait (on disk). + + - name: irq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: irq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: softirq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling software interrupts. + + - name: steal.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: steal.ticks + type: long + description: > + The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: cpu + type: group + description: > + `cpu` contains local CPU stats. + fields: + - name: cores + type: long + description: > + The number of CPU cores present on the host. The non-normalized + percentages will have a maximum value of `100% * cores`. The + normalized percentages already take this value into account and have + a maximum value of 100%. + + # Percentages + - name: user.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. On multi-core systems, + you can have percentages that are greater than 100%. For example, if 3 + cores are at 60% use, then the `system.cpu.user.pct` will be 180%. + + - name: system.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: nice.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: idle.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: iowait.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: irq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: steal.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: total.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in non-idle state. + + # Normalized Percentages + - name: user.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in user space. + + - name: system.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in kernel space. + + - name: nice.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent on low-priority processes. + + - name: idle.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent idle. + + - name: iowait.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in wait (on disk). + + - name: irq.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent servicing and handling software interrupts. + + - name: steal.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: total.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent in non-idle state. + + + # Ticks + - name: user.ticks + type: long + description: > + The amount of CPU time spent in user space. + + - name: system.ticks + type: long + description: > + The amount of CPU time spent in kernel space. + + - name: nice.ticks + type: long + description: > + The amount of CPU time spent on low-priority processes. + + - name: idle.ticks + type: long + description: > + The amount of CPU time spent idle. + + - name: iowait.ticks + type: long + description: > + The amount of CPU time spent in wait (on disk). + + - name: irq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling hardware interrupts. + + - name: softirq.ticks + type: long + description: > + The amount of CPU time spent servicing and handling software interrupts. + + - name: steal.ticks + type: long + description: > + The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor. + Available only on Unix. + + - name: diskio + type: group + description: > + `disk` contains disk IO metrics collected from the operating system. + fields: + - name: name + type: keyword + example: sda1 + description: > + The disk name. + + - name: serial_number + type: keyword + description: > + The disk's serial number. This may not be provided by all operating + systems. + + - name: read.count + type: long + description: > + The total number of reads completed successfully. + + - name: write.count + type: long + description: > + The total number of writes completed successfully. + + - name: read.bytes + type: long + format: bytes + description: > + The total number of bytes read successfully. On Linux this is + the number of sectors read multiplied by an assumed sector size of 512. + + - name: write.bytes + type: long + format: bytes + description: > + The total number of bytes written successfully. On Linux this is + the number of sectors written multiplied by an assumed sector size of + 512. + + - name: read.time + type: long + description: > + The total number of milliseconds spent by all reads. + + - name: write.time + type: long + description: > + The total number of milliseconds spent by all writes. + + - name: io.time + type: long + description: > + The total number of of milliseconds spent doing I/Os. + + - name: iostat.read.request.merges_per_sec + type: float + description: > + The number of read requests merged per second that were queued to the device. + + - name: iostat.write.request.merges_per_sec + type: float + description: > + The number of write requests merged per second that were queued to the device. + + - name: iostat.read.request.per_sec + type: float + description: > + The number of read requests that were issued to the device per second + + - name: iostat.write.request.per_sec + type: float + description: > + The number of write requests that were issued to the device per second + + - name: iostat.read.per_sec.bytes + type: float + description: > + The number of Bytes read from the device per second. + format: bytes + + - name: iostat.write.per_sec.bytes + type: float + description: > + The number of Bytes write from the device per second. + format: bytes + + - name: iostat.request.avg_size + type: float + description: > + The average size (in sectors) of the requests that were issued to the device. + + - name: iostat.queue.avg_size + type: float + description: > + The average queue length of the requests that were issued to the device. + + - name: iostat.await + type: float + description: > + The average queue length of the requests that were issued to the device. + + - name: iostat.service_time + type: float + description: > + The average service time (in milliseconds) for I/O requests that were issued to the device. + + - name: iostat.busy + type: float + description: > + Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. + + - name: filesystem + type: group + description: > + `filesystem` contains local filesystem stats. + fields: + - name: available + type: long + format: bytes + description: > + The disk space available to an unprivileged user in bytes. + - name: device_name + type: keyword + description: > + The disk name. For example: `/dev/disk1` + - name: type + type: keyword + description: > + The disk type. For example: `ext4` + - name: mount_point + type: keyword + description: > + The mounting point. For example: `/` + - name: files + type: long + description: > + The total number of file nodes in the file system. + - name: free + type: long + format: bytes + description: > + The disk space available in bytes. + - name: free_files + type: long + description: > + The number of free file nodes in the file system. + - name: total + type: long + format: bytes + description: > + The total disk space in bytes. + - name: used.bytes + type: long + format: bytes + description: > + The used disk space in bytes. + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used disk space. + + + + - name: fsstat + type: group + description: > + `system.fsstat` contains filesystem metrics aggregated from all mounted + filesystems, similar with what `df -a` prints out. + fields: + - name: count + type: long + description: Number of file systems found. + - name: total_files + type: long + description: Total number of files. + - name: total_size + format: bytes + type: group + description: Nested file system docs. + fields: + - name: free + type: long + format: bytes + description: > + Total free space. + - name: used + type: long + format: bytes + description: > + Total used space. + - name: total + type: long + format: bytes + description: > + Total space (used plus free). + + - name: load + type: group + description: > + CPU load averages. + fields: + - name: "1" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last minute. + - name: "5" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 5 minutes. + - name: "15" + type: scaled_float + scaling_factor: 100 + description: > + Load average for the last 15 minutes. + + - name: "norm.1" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last minute divided by the number of cores. + + - name: "norm.5" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last 5 minutes divided by the number of cores. + + - name: "norm.15" + type: scaled_float + scaling_factor: 100 + description: > + Load for the last 15 minutes divided by the number of cores. + + - name: "cores" + type: long + description: > + The number of CPU cores present on the host. + + - name: memory + type: group + description: > + `memory` contains local memory stats. + fields: + - name: total + type: long + format: bytes + description: > + Total memory. + + - name: used.bytes + type: long + format: bytes + description: > + Used memory. + + - name: free + type: long + format: bytes + description: > + The total amount of free memory in bytes. This value does not include memory consumed by system caches and + buffers (see system.memory.actual.free). + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used memory. + + - name: actual + type: group + description: > + Actual memory used and free. + fields: + + - name: used.bytes + type: long + format: bytes + description: > + Actual used memory in bytes. It represents the difference between the total and the available memory. The + available memory depends on the OS. For more details, please check `system.actual.free`. + + - name: free + type: long + format: bytes + description: > + Actual free memory in bytes. It is calculated based on the OS. On Linux it consists of the free memory + plus caches and buffers. On OSX it is a sum of free memory and the inactive memory. On Windows, it is equal + to `system.memory.free`. + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of actual used memory. + + - name: swap + type: group + prefix: "[float]" + description: This group contains statistics related to the swap memory usage on the system. + fields: + - name: total + type: long + format: bytes + description: > + Total swap memory. + + - name: used.bytes + type: long + format: bytes + description: > + Used swap memory. + + - name: free + type: long + format: bytes + description: > + Available swap memory. + + - name: used.pct + type: scaled_float + format: percent + description: > + The percentage of used swap memory. + + - name: network + type: group + description: > + `network` contains network IO metrics for a single network interface. + fields: + - name: name + type: keyword + example: eth0 + description: > + The network interface name. + + - name: out.bytes + type: long + format: bytes + description: > + The number of bytes sent. + + - name: in.bytes + type: long + format: bytes + description: > + The number of bytes received. + + - name: out.packets + type: long + description: > + The number of packets sent. + + - name: in.packets + type: long + description: > + The number or packets received. + + - name: in.errors + type: long + description: > + The number of errors while receiving. + + - name: out.errors + type: long + description: > + The number of errors while sending. + + - name: in.dropped + type: long + description: > + The number of incoming packets that were dropped. + + - name: out.dropped + type: long + description: > + The number of outgoing packets that were dropped. This value is always + 0 on Darwin and BSD because it is not reported by the operating system. + + - name: process + type: group + description: > + `process` contains process metadata, CPU metrics, and memory metrics. + fields: + - name: name + type: keyword + description: > + The process name. + - name: state + type: keyword + description: > + The process state. For example: "running". + - name: pid + type: long + description: > + The process pid. + - name: ppid + type: long + description: > + The process parent pid. + - name: pgid + type: long + description: > + The process group id. + - name: cmdline + type: keyword + description: > + The full command-line used to start the process, including the + arguments separated by space. + - name: username + type: keyword + description: > + The username of the user that created the process. If the username + cannot be determined, the field will contain the user's + numeric identifier (UID). On Windows, this field includes the user's + domain and is formatted as `domain\username`. + - name: cwd + type: keyword + description: > + The current working directory of the process. This field is only + available on Linux. + - name: env + type: object + object_type: keyword + description: > + The environment variables used to start the process. The data is + available on FreeBSD, Linux, and OS X. + - name: cpu + type: group + prefix: "[float]" + description: CPU-specific statistics per process. + fields: + - name: user + type: long + description: > + The amount of CPU time the process spent in user space. + - name: total.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent by the process since the last update. Its value is similar to the + %CPU value of the process displayed by the top command on Unix systems. + - name: total.norm.pct + type: scaled_float + format: percent + description: > + The percentage of CPU time spent by the process since the last event. + This value is normalized by the number of CPU cores and it ranges + from 0 to 100%. + - name: system + type: long + description: > + The amount of CPU time the process spent in kernel space. + - name: total.ticks + type: long + description: > + The total CPU time spent by the process. + - name: start_time + type: date + description: > + The time when the process was started. + - name: memory + type: group + description: Memory-specific statistics per process. + prefix: "[float]" + fields: + - name: size + type: long + format: bytes + description: > + The total virtual memory the process has. + - name: rss.bytes + type: long + format: bytes + description: > + The Resident Set Size. The amount of memory the process occupied in main memory (RAM). + - name: rss.pct + type: scaled_float + format: percent + description: > + The percentage of memory the process occupied in main memory (RAM). + - name: share + type: long + format: bytes + description: > + The shared memory the process uses. + - name: fd + type: group + description: > + File descriptor usage metrics. This set of metrics is available for + Linux and FreeBSD. + prefix: "[float]" + fields: + - name: open + type: long + description: The number of file descriptors open by the process. + - name: limit.soft + type: long + description: > + The soft limit on the number of file descriptors opened by the + process. The soft limit can be changed by the process at any time. + - name: limit.hard + type: long + description: > + The hard limit on the number of file descriptors opened by the + process. The hard limit can only be raised by root. + - name: cgroup + type: group + description: > + Metrics and limits from the cgroup of which the task is a member. + cgroup metrics are reported when the process has membership in a + non-root cgroup. These metrics are only available on Linux. + fields: + - name: id + type: keyword + description: > + The ID common to all cgroups associated with this task. + If there isn't a common ID used by all cgroups this field will be + absent. + + - name: path + type: keyword + description: > + The path to the cgroup relative to the cgroup subsystem's mountpoint. + If there isn't a common path used by all cgroups this field will be + absent. + + - name: cpu + type: group + description: > + The cpu subsystem schedules CPU access for tasks in the cgroup. + Access can be controlled by two separate schedulers, CFS and RT. + CFS stands for completely fair scheduler which proportionally + divides the CPU time between cgroups based on weight. RT stands for + real time scheduler which sets a maximum amount of CPU time that + processes in the cgroup can consume during a given period. + + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's + mountpoint. + + - name: cfs.period.us + type: long + description: > + Period of time in microseconds for how regularly a + cgroup's access to CPU resources should be reallocated. + + - name: cfs.quota.us + type: long + description: > + Total amount of time in microseconds for which all + tasks in a cgroup can run during one period (as defined by + cfs.period.us). + + - name: cfs.shares + type: long + description: > + An integer value that specifies a relative share of CPU time + available to the tasks in a cgroup. The value specified in the + cpu.shares file must be 2 or higher. + + - name: rt.period.us + type: long + description: > + Period of time in microseconds for how regularly a cgroup's + access to CPU resources is reallocated. + + - name: rt.runtime.us + type: long + description: > + Period of time in microseconds for the longest continuous period + in which the tasks in a cgroup have access to CPU resources. + + - name: stats.periods + type: long + description: > + Number of period intervals (as specified in cpu.cfs.period.us) + that have elapsed. + + - name: stats.throttled.periods + type: long + description: > + Number of times tasks in a cgroup have been throttled (that is, + not allowed to run because they have exhausted all of the + available time as specified by their quota). + + - name: stats.throttled.ns + type: long + description: > + The total time duration (in nanoseconds) for which tasks in a + cgroup have been throttled. + + - name: cpuacct + type: group + description: CPU accounting metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's + mountpoint. + + - name: total.ns + type: long + description: > + Total CPU time in nanoseconds consumed by all tasks in the + cgroup. + + - name: stats.user.ns + type: long + description: CPU time consumed by tasks in user mode. + + - name: stats.system.ns + type: long + description: CPU time consumed by tasks in user (kernel) mode. + + - name: percpu + type: object + object_type: long + description: > + CPU time (in nanoseconds) consumed on each CPU by all tasks in + this cgroup. + + - name: memory + type: group + description: Memory limits and metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystem's mountpoint. + + - name: mem.usage.bytes + type: long + format: bytes + description: > + Total memory usage by processes in the cgroup (in bytes). + + - name: mem.usage.max.bytes + type: long + format: bytes + description: > + The maximum memory used by processes in the cgroup (in bytes). + + - name: mem.limit.bytes + type: long + format: bytes + description: > + The maximum amount of user memory in bytes (including file + cache) that tasks in the cgroup are allowed to use. + + - name: mem.failures + type: long + description: > + The number of times that the memory limit (mem.limit.bytes) was + reached. + + - name: memsw.usage.bytes + type: long + format: bytes + description: > + The sum of current memory usage plus swap space used by + processes in the cgroup (in bytes). + + - name: memsw.usage.max.bytes + type: long + format: bytes + description: > + The maximum amount of memory and swap space used by processes in + the cgroup (in bytes). + + - name: memsw.limit.bytes + type: long + format: bytes + description: > + The maximum amount for the sum of memory and swap usage + that tasks in the cgroup are allowed to use. + + - name: memsw.failures + type: long + description: > + The number of times that the memory plus swap space limit + (memsw.limit.bytes) was reached. + + - name: kmem.usage.bytes + type: long + format: bytes + description: > + Total kernel memory usage by processes in the cgroup (in bytes). + + - name: kmem.usage.max.bytes + type: long + format: bytes + description: > + The maximum kernel memory used by processes in the cgroup (in + bytes). + + - name: kmem.limit.bytes + type: long + format: bytes + description: > + The maximum amount of kernel memory that tasks in the cgroup are + allowed to use. + + - name: kmem.failures + type: long + description: > + The number of times that the memory limit (kmem.limit.bytes) was + reached. + + - name: kmem_tcp.usage.bytes + type: long + format: bytes + description: > + Total memory usage for TCP buffers in bytes. + + - name: kmem_tcp.usage.max.bytes + type: long + format: bytes + description: > + The maximum memory used for TCP buffers by processes in the + cgroup (in bytes). + + - name: kmem_tcp.limit.bytes + type: long + format: bytes + description: > + The maximum amount of memory for TCP buffers that tasks in the + cgroup are allowed to use. + + - name: kmem_tcp.failures + type: long + description: > + The number of times that the memory limit (kmem_tcp.limit.bytes) + was reached. + + - name: stats.active_anon.bytes + type: long + format: bytes + description: > + Anonymous and swap cache on active least-recently-used (LRU) + list, including tmpfs (shmem), in bytes. + + - name: stats.active_file.bytes + type: long + format: bytes + description: File-backed memory on active LRU list, in bytes. + + - name: stats.cache.bytes + type: long + format: bytes + description: Page cache, including tmpfs (shmem), in bytes. + + - name: stats.hierarchical_memory_limit.bytes + type: long + format: bytes + description: > + Memory limit for the hierarchy that contains the memory cgroup, + in bytes. + + - name: stats.hierarchical_memsw_limit.bytes + type: long + format: bytes + description: > + Memory plus swap limit for the hierarchy that contains the + memory cgroup, in bytes. + + - name: stats.inactive_anon.bytes + type: long + format: bytes + description: > + Anonymous and swap cache on inactive LRU list, including tmpfs + (shmem), in bytes + + - name: stats.inactive_file.bytes + type: long + format: bytes + description: > + File-backed memory on inactive LRU list, in bytes. + + - name: stats.mapped_file.bytes + type: long + format: bytes + description: > + Size of memory-mapped mapped files, including tmpfs (shmem), + in bytes. + + - name: stats.page_faults + type: long + description: > + Number of times that a process in the cgroup triggered a page + fault. + + - name: stats.major_page_faults + type: long + description: > + Number of times that a process in the cgroup triggered a major + fault. "Major" faults happen when the kernel actually has to + read the data from disk. + + - name: stats.pages_in + type: long + description: > + Number of pages paged into memory. This is a counter. + + - name: stats.pages_out + type: long + description: > + Number of pages paged out of memory. This is a counter. + + - name: stats.rss.bytes + type: long + format: bytes + description: > + Anonymous and swap cache (includes transparent hugepages), not + including tmpfs (shmem), in bytes. + + - name: stats.rss_huge.bytes + type: long + format: bytes + description: > + Number of bytes of anonymous transparent hugepages. + + - name: stats.swap.bytes + type: long + format: bytes + description: > + Swap usage, in bytes. + + - name: stats.unevictable.bytes + type: long + format: bytes + description: > + Memory that cannot be reclaimed, in bytes. + + - name: blkio + type: group + description: Block IO metrics. + fields: + - name: id + type: keyword + description: ID of the cgroup. + + - name: path + type: keyword + description: > + Path to the cgroup relative to the cgroup subsystems mountpoint. + + - name: total.bytes + type: long + format: bytes + description: > + Total number of bytes transferred to and from all block devices + by processes in the cgroup. + + - name: total.ios + type: long + description: > + Total number of I/O operations performed on all devices + by processes in the cgroup as seen by the throttling policy. + + - name: process.summary + title: Process Summary + type: group + description: > + Summary metrics for the processes running on the host. + fields: + - name: total + type: long + description: > + Total number of processes on this host. + - name: running + type: long + description: > + Number of running processes on this host. + - name: idle + type: long + description: > + Number of idle processes on this host. + - name: sleeping + type: long + description: > + Number of sleeping processes on this host. + - name: stopped + type: long + description: > + Number of stopped processes on this host. + - name: zombie + type: long + description: > + Number of zombie processes on this host. + - name: unknown + type: long + description: > + Number of processes for which the state couldn't be retrieved or is unknown. + + - name: socket + type: group + description: > + TCP sockets that are active. + fields: + - name: direction + type: keyword + example: incoming + description: > + How the socket was initiated. Possible values are incoming, outgoing, + or listening. + + - name: family + type: keyword + example: ipv4 + description: > + Address family. + + - name: local.ip + type: ip + example: 192.0.2.1 or 2001:0DB8:ABED:8536::1 + description: > + Local IP address. This can be an IPv4 or IPv6 address. + + - name: local.port + type: long + example: 22 + description: > + Local port. + + - name: remote.ip + type: ip + example: 192.0.2.1 or 2001:0DB8:ABED:8536::1 + description: > + Remote IP address. This can be an IPv4 or IPv6 address. + + - name: remote.port + type: long + example: 22 + description: > + Remote port. + + - name: remote.host + type: keyword + example: 76-211-117-36.nw.example.com. + description: > + PTR record associated with the remote IP. It is obtained via reverse + IP lookup. + + - name: remote.etld_plus_one + type: keyword + example: example.com. + description: > + The effective top-level domain (eTLD) of the remote host plus one more + label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". + The data for determining the eTLD comes from an embedded copy of the data + from http://publicsuffix.org. + + - name: remote.host_error + type: keyword + description: > + Error describing the cause of the reverse lookup failure. + + - name: process.pid + type: long + description: > + ID of the process that opened the socket. + + - name: process.command + type: keyword + description: > + Name of the command (limited to 20 chars by the OS). + + - name: process.cmdline + type: keyword + description: > + + - name: process.exe + type: keyword + description: > + Absolute path to the executable. + + - name: user.id + type: long + description: > + UID of the user running the process. + + - name: user.name + type: keyword + description: > + Name of the user running the process. + + - name: uptime + type: group + description: > + `uptime` contains the operating system uptime metric. + fields: + - name: duration.ms + type: long + format: duration + input_format: milliseconds + description: > + The OS uptime in milliseconds. + +- key: vsphere + title: "vSphere" + description: > + vSphere module + fields: + - name: vsphere + type: group + description: > + fields: + - name: datastore + type: group + description: > + datastore + fields: + - name: datacenter + type: keyword + description: > + Datacenter name + - name: name + type: keyword + description: > + Datastore name + - name: fstype + type: keyword + description: > + Filesystem type + - name: capacity.total.bytes + type: long + description: > + Total bytes of the datastore + format: bytes + - name: capacity.free.bytes + type: long + description: > + Free bytes of the datastore + format: bytes + - name: capacity.used.bytes + type: long + description: > + Used bytes of the datastore + format: bytes + - name: capacity.used.pct + type: long + description: > + Used percent of the datastore + format: percent + + + - name: host + type: group + description: > + host + fields: + - name: datacenter + type: keyword + description: > + Datacenter name + - name: name + type: keyword + description: > + Host name + - name: cpu.used.mhz + type: long + description: > + Used CPU in Mhz + - name: cpu.total.mhz + type: long + description: > + Total CPU in Mhz + - name: cpu.free.mhz + type: long + description: > + Free CPU in Mhz + - name: memory.used.bytes + type: long + description: > + Used Memory in bytes + format: bytes + - name: memory.total.bytes + type: long + description: > + Total Memory in bytes + format: bytes + - name: memory.free.bytes + type: long + description: > + Free Memory in bytes + format: bytes + + - name: virtualmachine + type: group + description: > + virtualmachine + fields: + - name: datacenter + type: keyword + description: > + Datacenter name + - name: name + type: keyword + description: > + Virtual Machine name + - name: cpu.used.mhz + type: long + description: > + Used CPU in Mhz + - name: memory.used.guest.bytes + type: long + description: > + Used Memory of Guest in bytes + format: bytes + - name: memory.used.host.bytes + type: long + description: > + Used Memory of Host in bytes + format: bytes + - name: memory.total.guest.bytes + type: long + description: > + Total Memory of Guest in bytes + format: bytes + - name: memory.free.guest.bytes + type: long + description: > + Free Memory of Guest in bytes + format: bytes + - name: custom_fields + type: object + object_type: keyword + descripton: > + Custom fields + +- key: windows + title: "Windows" + description: > + beta[] + Module for Windows + short_config: false + fields: + - name: windows + type: group + description: > + fields: + +- key: zookeeper + title: "ZooKeeper" + description: > + ZooKeeper metrics collected by the four-letter monitoring commands. + short_config: false + fields: + - name: zookeeper + type: group + description: > + `zookeeper` contains the metrics reported by ZooKeeper + commands. + fields: + - name: mntr + type: group + description: > + `mntr` contains the metrics reported by the four-letter `mntr` + command. + fields: + - name: hostname + type: keyword + description: > + ZooKeeper hostname. + - name: approximate_data_size + type: long + description: > + Approximate size of ZooKeeper data. + - name: latency.avg + type: long + description: > + Average latency between ensemble hosts in milliseconds. + - name: ephemerals_count + type: long + description: > + Number of ephemeral znodes. + - name: followers + type: long + description: > + Number of followers seen by the current host. + - name: max_file_descriptor_count + type: long + description: > + Maximum number of file descriptors allowed for the ZooKeeper process. + - name: latency.max + type: long + description: > + Maximum latency in milliseconds. + - name: latency.min + type: long + description: > + Minimum latency in milliseconds. + - name: num_alive_connections + type: long + description: > + Number of connections to ZooKeeper that are currently alive. + - name: open_file_descriptor_count + type: long + description: > + Number of file descriptors open by the ZooKeeper process. + - name: outstanding_requests + type: long + description: > + Number of outstanding requests that need to be processed by the cluster. + - name: packets.received + type: long + description: > + Number of ZooKeeper network packets received. + - name: packets.sent + type: long + description: > + Number of ZooKeeper network packets sent. + - name: pending_syncs + type: long + description: > + Number of pending syncs to carry out to ZooKeeper ensemble followers. + - name: server_state + type: keyword + description: > + Role in the ZooKeeper ensemble. + - name: synced_followers + type: long + description: > + Number of synced followers reported when a node server_state is leader. + - name: version + type: keyword + description: > + ZooKeeper version and build string reported. + - name: watch_count + type: long + description: > + Number of watches currently set on the local ZooKeeper process. + - name: znode_count + type: long + description: > + Number of znodes reported by the local ZooKeeper process. + + diff --git a/libbeat/kibana/testdata/metricbeat-5x-old.json b/libbeat/kibana/testdata/extensive/metricbeat-5x.json similarity index 100% rename from libbeat/kibana/testdata/metricbeat-5x-old.json rename to libbeat/kibana/testdata/extensive/metricbeat-5x.json diff --git a/libbeat/kibana/testdata/metricbeat-default-old.json b/libbeat/kibana/testdata/extensive/metricbeat-default.json similarity index 100% rename from libbeat/kibana/testdata/metricbeat-default-old.json rename to libbeat/kibana/testdata/extensive/metricbeat-default.json diff --git a/libbeat/kibana/testdata/fields.yml b/libbeat/kibana/testdata/fields.yml index 76154b7a6e1..a935109a0f7 100644 --- a/libbeat/kibana/testdata/fields.yml +++ b/libbeat/kibana/testdata/fields.yml @@ -1,10 +1,6 @@ - -- key: beat - title: Beat - description: > - Contains common beat fields available in all event types. +- key: test + title: Test fields.yml fields: - - name: group_disabled type: group enabled: false @@ -12,6854 +8,11 @@ - name: message type: text - - name: beat.name - description: > - The name of the Beat sending the log messages. If the Beat name is - set in the configuration file, then that value is used. If it is not - set, the hostname is used. To set the Beat name, use the `name` - option in the configuration file. - - name: beat.hostname - description: > - The hostname as returned by the operating system on which the Beat is - running. - - name: beat.timezone - description: > - The timezone as returned by the operating system on which the Beat is - running. - - name: beat.version - description: > - The version of the beat that generated this event. - - - name: "@timestamp" - type: date - required: true - format: date - example: August 26th 2016, 12:35:53.332 - description: > - The timestamp when the event log record was generated. - - - name: tags - description: > - Arbitrary tags that can be set per Beat and per transaction - type. - - - name: fields - type: object - object_type: keyword - description: > - Contains user configurable fields. - - - name: error - type: group - description: > - Error fields containing additional info in case of errors. - fields: - - name: message - type: text - description: > - Error message. - - name: code - type: long - description: > - Error code. - - name: type - type: keyword - description: > - Error type. -- key: cloud - title: Cloud provider metadata - description: > - Metadata from cloud providers added by the add_cloud_metadata processor. - fields: - - - name: meta.cloud.provider - example: ec2 - description: > - Name of the cloud provider. Possible values are ec2, gce, or digitalocean. - - - name: meta.cloud.instance_id - description: > - Instance ID of the host machine. - - - name: meta.cloud.instance_name - description: > - Instance name of the host machine. - - - name: meta.cloud.machine_type - example: t2.medium - description: > - Machine type of the host machine. - - - name: meta.cloud.availability_zone - example: us-east-1c - description: > - Availability zone in which this host is running. - - - name: meta.cloud.project_id - example: project-x - description: > - Name of the project in Google Cloud. - - - name: meta.cloud.region - description: > - Region in which this host is running. -- key: docker - title: Docker - description: > - beta[] - - Docker stats collected from Docker. - short_config: false - anchor: docker-processor - fields: - - name: docker - type: group - fields: - - name: container.id - type: keyword - description: > - Unique container id. - - name: container.image - type: keyword - description: > - Name of the image the container was built on. - - name: container.name - type: keyword - description: > - Container name. - - name: container.labels - type: object - object_type: keyword - description: > - Image labels. -- key: kubernetes - title: Kubernetes - description: > - beta[] - - Kubernetes metadata added by the kubernetes processor - short_config: false - anchor: kubernetes-processor - fields: - - name: kubernetes - type: group - fields: - - name: pod.name - type: keyword - description: > - Kubernetes pod name - - - name: namespace - type: keyword - description: > - Kubernetes namespace - - - name: labels - type: object - description: > - Kubernetes labels map - - - name: annotations - type: object - description: > - Kubernetes annotations map - - - name: container.name - type: keyword - description: > - Kubernetes container name - - - name: container.image - type: keyword - description: > - Kubernetes container image -- key: common - title: Common - description: > - Contains common fields available in all event types. - fields: - - - name: metricset.module - description: > - The name of the module that generated the event. - - - name: metricset.name - description: > - The name of the metricset that generated the event. - - - name: metricset.host - description: > - Hostname of the machine from which the metricset was collected. This - field may not be present when the data was collected locally. - - - name: metricset.rtt - type: long - required: true - description: > - Event round trip time in microseconds. - - - name: metricset.namespace - type: keyword - description: > - Namespace of dynamic metricsets. - - - name: type - required: true - example: metricsets - description: > - The document type. Always set to "metricsets". - -- key: aerospike - title: "Aerospike" - description: > - experimental[] - - Aerospike module - fields: - - name: aerospike - type: group - description: > - fields: - - name: namespace - type: group - description: > - namespace - fields: - - name: client - type: group - description: > - Client stats. - fields: - - name: delete - type: group - description: > - Client delete transactions stats. - fields: - - name: error - type: long - description: > - Number of client delete transactions that failed with an error. - - name: not_found - type: long - description: > - Number of client delete transactions that resulted in a not found. - - name: success - type: long - description: > - Number of successful client delete transactions. - - name: timeout - type: long - description: > - Number of client delete transactions that timed out. - - name: read - type: group - description: > - Client read transactions stats. - fields: - - name: error - type: long - description: > - Number of client read transaction errors. - - name: not_found - type: long - description: > - Number of client read transaction that resulted in not found. - - name: success - type: long - description: > - Number of successful client read transactions. - - name: timeout - type: long - description: > - Number of client read transaction that timed out. - - name: write - type: group - description: > - Client write transactions stats. - fields: - - name: error - type: long - description: > - Number of client write transactions that failed with an error. - - name: success - type: long - description: > - Number of successful client write transactions. - - name: timeout - type: long - description: > - Number of client write transactions that timed out. - - name: device - type: group - description: > - Disk storage stats - fields: - - name: available.pct - type: scaled_float - format: percent - description: > - Measures the minimum contiguous disk space across all disks in a namespace. - - name: free.pct - type: scaled_float - format: percent - description: > - Percentage of disk capacity free for this namespace. - - name: total.bytes - type: long - format: bytes - description: > - Total bytes of disk space allocated to this namespace on this node. - - name: used.bytes - type: long - format: bytes - description: > - Total bytes of disk space used by this namespace on this node. - - name: hwm_breached - type: boolean - description: > - If true, Aerospike has breached 'high-water-[disk|memory]-pct' for this namespace. - - name: memory - type: group - description: > - Memory storage stats. - fields: - - name: free.pct - type: scaled_float - format: percent - description: > - Percentage of memory capacity free for this namespace on this node. - - name: used.data.bytes - type: long - format: bytes - description: > - Amount of memory occupied by data for this namespace on this node. - - name: used.index.bytes - type: long - format: bytes - description: > - Amount of memory occupied by the index for this namespace on this node. - - name: used.sindex.bytes - type: long - format: bytes - description: > - Amount of memory occupied by secondary indexes for this namespace on this node. - - name: used.total.bytes - type: long - format: bytes - description: > - Total bytes of memory used by this namespace on this node. - - name: name - type: keyword - description: > - Namespace name - - name: node.host - type: keyword - desription: > - Node host - - name: node.name - type: keyword - description: > - Node name - - name: objects - type: group - description: > - Records stats. - fields: - - name: master - type: long - description: > - Number of records on this node which are active masters. - - name: total - type: long - description: > - Number of records in this namespace for this node. - - name: stop_writes - type: boolean - description: > - If true this namespace is currently not allowing writes. - -- key: apache - title: "Apache" - description: > - Apache HTTPD server metricsets collected from the Apache web server. - short_config: false - fields: - - name: apache - type: group - description: > - `apache` contains the metrics that were scraped from Apache. - fields: - - name: status - type: group - description: > - `status` contains the metrics that were scraped from the Apache status page. - fields: - - name: hostname - type: keyword - description: > - Apache hostname. - - name: total_accesses - type: long - description: > - Total number of access requests. - - name: total_kbytes - type: long - description: > - Total number of kilobytes served. - - name: requests_per_sec - type: scaled_float - description: > - Requests per second. - - name: bytes_per_sec - type: scaled_float - description: > - Bytes per second. - - name: bytes_per_request - type: scaled_float - description: > - Bytes per request. - - name: workers.busy - type: long - description: > - Number of busy workers. - - name: workers.idle - type: long - description: > - Number of idle workers. - - name: uptime - type: group - description: > - Uptime stats. - fields: - - name: server_uptime - type: long - description: > - Server uptime in seconds. - - name: uptime - type: long - description: > - Server uptime. - - name: cpu - type: group - description: > - CPU stats. - fields: - - name: load - type: scaled_float - description: > - CPU Load. - - name: user - type: scaled_float - description: > - CPU user load. - - name: system - type: scaled_float - description: > - System cpu. - - name: children_user - type: scaled_float - description: > - CPU of children user. - - name: children_system - type: scaled_float - description: > - CPU of children system. - - name: connections - type: group - description: > - Connection stats. - fields: - - name: total - type: long - description: > - Total connections. - - name: async.writing - type: long - description: > - Async connection writing. - - name: async.keep_alive - type: long - description: > - Async keeped alive connections. - - name: async.closing - type: long - description: > - Async closed connections. - - name: load - type: group - description: > - Load averages. - fields: - - name: "1" - type: scaled_float - scaling_factor: 100 - description: > - Load average for the last minute. - - name: "5" - type: scaled_float - scaling_factor: 100 - description: > - Load average for the last 5 minutes. - - name: "15" - type: scaled_float - scaling_factor: 100 - description: > - Load average for the last 15 minutes. - - name: scoreboard - type: group - description: > - Scoreboard metrics. - fields: - - name: starting_up - type: long - description: > - Starting up. - - name: reading_request - type: long - description: > - Reading requests. - - name: sending_reply - type: long - description: > - Sending Reply. - - name: keepalive - type: long - description: > - Keep alive. - - name: dns_lookup - type: long - description: > - Dns Lookups. - - name: closing_connection - type: long - description: > - Closing connections. - - name: logging - type: long - description: > - Logging - - name: gracefully_finishing - type: long - description: > - Gracefully finishing. - - name: idle_cleanup - type: long - description: > - Idle cleanups. - - name: open_slot - type: long - description: > - Open slots. - - name: waiting_for_connection - type: long - description: > - Waiting for connections. - - name: total - type: long - description: > - Total. - -- key: ceph - title: "Ceph" - description: > - beta[] - - Ceph module - short_config: false - fields: - - name: ceph - type: group - description: > - `ceph` contains the metrics that were scraped from CEPH. - fields: - - name: cluster_disk - type: group - description: > - cluster_disk - fields: - - name: available.bytes - type: long - description: > - Available bytes of the cluster - format: bytes - - name: total.bytes - type: long - description: > - Total bytes of the cluster - format: bytes - - name: used.bytes - type: long - description: > - Used bytes of the cluster - format: bytes - - - name: cluster_health - type: group - description: > - cluster_health - fields: - - name: overall_status - type: keyword - description: > - Overall status of the cluster - - name: timechecks.epoch - type: long - description: > - Map version - - name: timechecks.round.value - type: long - description: > - timecheck round - - name: timechecks.round.status - type: keyword - description: > - Status of the round - - - name: cluster_status - type: group - description: > - cluster_status - fields: - - name: version - type: long - description: > - Ceph Status version - - name: traffic.read_bytes - type: long - format: bytes - description: > - Cluster read throughput per second - - name: traffic.write_bytes - type: long - format: bytes - description: > - Cluster write throughput per second - - name: traffic.read_op_per_sec - type: long - description: > - Cluster read iops per second - - name: traffic.write_op_per_sec - type: long - description: > - Cluster write iops per second - - name: misplace.total - type: long - description: > - Cluster misplace pg number - - name: misplace.objects - type: long - description: > - Cluster misplace objects number - - name: misplace.ratio - type: scaled_float - format: percent - description: > - Cluster misplace ratio - - name: degraded.total - type: long - description: > - Cluster degraded pg number - - name: degraded.objects - type: long - description: > - Cluster degraded objects number - - name: degraded.ratio - type: scaled_float - format: percent - description: > - Cluster degraded ratio - - name: pg.data_bytes - type: long - format: bytes - description: > - Cluster pg data bytes - - name: pg.avail_bytes - type: long - format: bytes - description: > - Cluster available bytes - - name: pg.total_bytes - type: long - format: bytes - description: > - Cluster total bytes - - name: pg.used_bytes - type: long - format: bytes - description: > - Cluster used bytes - - name: pg_state.state_name - type: long - description: > - Pg state description - - name: pg_state.count - type: long - description: > - Shows how many pgs are in state of pg_state.state_name - - name: pg_state.version - type: long - description: > - Cluster status version - - name: osd.full - type: boolean - description: > - Is osd full - - name: osd.nearfull - type: boolean - description: > - Is osd near full - - name: osd.num_osds - type: long - description: > - Shows how many osds in the cluster - - name: osd.num_up_osds - type: long - description: > - Shows how many osds are on the state of UP - - name: osd.num_in_osds - type: long - description: > - Shows how many osds are on the state of IN - - name: osd.num_in_osds2 - type: long - description: > - Shows how many osds are on the state of REMAPPED - - name: osd.epoch - type: long - description: > - epoch number - - - name: monitor_health - type: group - description: > - monitor_health stats data - fields: - - name: available.pct - type: long - description: > - Available percent of the MON - - name: health - type: keyword - description: > - Health of the MON - - name: available.kb - type: long - description: > - Available KB of the MON - - name: total.kb - type: long - description: > - Total KB of the MON - - name: used.kb - type: long - description: > - Used KB of the MON - - name: last_updated - type: date - description: > - Time when was updated - - name: name - type: keyword - description: > - Name of the MON - - name: store_stats.log.bytes - type: long - description: > - Log bytes of MON - format: bytes - - name: store_stats.misc.bytes - type: long - description: > - Misc bytes of MON - format: bytes - - name: store_stats.sst.bytes - type: long - description: > - SST bytes of MON - format: bytes - - name: store_stats.total.bytes - type: long - description: > - Total bytes of MON - format: bytes - - name: store_stats.last_updated - type: long - description: > - Last updated - - - name: pool_disk - type: group - description: > - pool_disk - fields: - - name: id - type: long - description: > - Id of the pool - - name: name - type: keyword - description: > - Name of the pool - - name: stats.available.bytes - type: long - description: > - Available bytes of the pool - format: bytes - - name: stats.objects - type: long - description: > - Number of objects of the pool - - name: stats.used.bytes - type: long - description: > - Used bytes of the pool - format: bytes - - name: stats.used.kb - type: long - description: > - Used kb of the pool + - name: long + type: long -- key: couchbase - title: "Couchbase" - description: > - beta[] - - Metrics collected from Couchbase servers. - short_config: false - fields: - - name: couchbase - type: group - description: > - `couchbase` contains the metrics that were scraped from Couchbase. - fields: - - name: bucket - type: group - description: > - Couchbase bucket metrics. - fields: - - name: name - type: keyword - description: > - Name of the bucket. - - name: type - type: keyword - description: > - Type of the bucket. - - name: data.used.bytes - format: bytes - type: long - description: > - Size of user data within buckets of the specified state that are resident in RAM. - - name: disk.fetches - type: long - description: > - Number of disk fetches. - - name: disk.used.bytes - format: bytes - type: long - description: > - Amount of disk used (bytes). - - name: memory.used.bytes - format: bytes - type: long - description: > - Amount of memory used by the bucket (bytes). - - name: quota.ram.bytes - format: bytes - type: long - description: > - Amount of RAM used by the bucket (bytes). - - name: quota.use.pct - format: percent - type: scaled_float - description: > - Percentage of RAM used (for active objects) against the configured bucket size (%). - - name: ops_per_sec - type: long - description: > - Number of operations per second. - - name: item_count - type: long - description: > - Number of items associated with the bucket. - - - name: cluster - type: group - description: > - Couchbase cluster metrics. - fields: - - name: hdd.free.bytes - format: bytes - type: long - description: > - Free hard drive space in the cluster (bytes). - - name: hdd.quota.total.bytes - format: bytes - type: long - description: > - Hard drive quota total for the cluster (bytes). - - name: hdd.total.bytes - format: bytes - type: long - description: > - Total hard drive space available to the cluster (bytes). - - name: hdd.used.value.bytes - format: bytes - type: long - description: > - Hard drive space used by the cluster (bytes). - - name: hdd.used.by_data.bytes - format: bytes - type: long - description: > - Hard drive space used by the data in the cluster (bytes). - - name: max_bucket_count - type: long - description: > - Max bucket count setting. - - name: quota.index_memory.mb - type: long - description: > - Memory quota setting for the Index service (Mbyte). - - name: quota.memory.mb - type: long - description: > - Memory quota setting for the cluster (Mbyte). - - name: ram.quota.total.value.bytes - format: bytes - type: long - description: > - RAM quota total for the cluster (bytes). - - name: ram.quota.total.per_node.bytes - format: bytes - type: long - description: > - RAM quota used by the current node in the cluster (bytes). - - name: ram.quota.used.value.bytes - format: bytes - type: long - description: > - RAM quota used by the cluster (bytes). - - name: ram.quota.used.per_node.bytes - format: bytes - type: long - description: > - Ram quota used by the current node in the cluster (bytes) - - name: ram.total.bytes - format: bytes - type: long - description: > - Total RAM available to cluster (bytes). - - name: ram.used.value.bytes - format: bytes - type: long - description: > - RAM used by the cluster (bytes). - - name: ram.used.by_data.bytes - format: bytes - type: long - description: > - RAM used by the data in the cluster (bytes). - - - name: node - type: group - description: > - Couchbase node metrics. - fields: - - name: cmd_get - type: long - description: > - Number of get commands - - name: couch.docs.disk_size.bytes - format: bytes - type: long - description: > - Amount of disk space used by Couch docs (bytes). - - name: couch.docs.data_size.bytes - format: bytes - type: long - description: > - Data size of Couch docs associated with a node (bytes). - - name: couch.spatial.data_size.bytes - type: long - description: > - Size of object data for spatial views (bytes). - - name: couch.spatial.disk_size.bytes - type: long - description: > - Amount of disk space used by spatial views (bytes). - - name: couch.views.disk_size.bytes - type: long - description: > - Amount of disk space used by Couch views (bytes). - - name: couch.views.data_size.bytes - type: long - description: > - Size of object data for Couch views (bytes). - - name: cpu_utilization_rate.pct - type: scaled_float - description: > - The CPU utilization rate (%). - - name: current_items.value - type: long - description: > - Number of current items. - - name: current_items.total - type: long - description: > - Total number of items associated with the node. - - name: ep_bg_fetched - type: long - description: > - Number of disk fetches performed since the server was started. - - name: get_hits - type: long - description: > - Number of get hits. - - name: hostname - type: keyword - description: > - The hostname of the node. - - name: mcd_memory.allocated.bytes - format: bytes - type: long - description: > - Amount of memcached memory allocated (bytes). - - name: mcd_memory.reserved.bytes - type: long - description: > - Amount of memcached memory reserved (bytes). - - name: memory.free.bytes - type: long - description: > - Amount of memory free for the node (bytes). - - name: memory.total.bytes - type: long - description: > - Total memory available to the node (bytes). - - name: memory.used.bytes - type: long - description: > - Memory used by the node (bytes). - - name: ops - type: long - description: > - Number of operations performed on Couchbase. - - name: swap.total.bytes - type: long - description: > - Total swap size allocated (bytes). - - name: swap.used.bytes - type: long - description: > - Amount of swap space used (bytes). - - name: uptime.sec - type: long - description: > - Time during which the node was in operation (sec). - - name: vb_replica_curr_items - type: long - description: > - Number of items/documents that are replicas. - -- key: docker - title: "Docker" - description: > - beta[] - - Docker stats collected from Docker. - short_config: false - fields: - - name: docker - type: group - description: > - Information and statistics about docker's running containers. - fields: - - name: container - type: group - description: > - Docker container metrics. - fields: - - name: command - type: keyword - description: > - Command that was executed in the Docker container. - - name: created - type: date - description: > - Date when the container was created. - - name: status - type: keyword - description: > - Container status. - - name: size - type: group - description: > - Container size metrics. - fields: - - name: root_fs - type: long - description: > - Total size of all the files in the container. - - name: rw - type: long - description: > - Size of the files that have been created or changed since creation. - - name: tags - type: array - description: > - Image tags. - - - name: cpu - type: group - description: > - Runtime CPU metrics. - fields: - - name: kernel.pct - type: scaled_float - format: percentage - description: > - The system kernel consumed by the Docker server. - - name: kernel.ticks - type: long - description: > - CPU kernel ticks. - - name: system.pct - type: scaled_float - format: percentage - description: > - - name: system.ticks - type: long - description: > - CPU system ticks. - - name: user.pct - type: scaled_float - format: percentage - description: > - - name: user.ticks - type: long - description: > - CPU user ticks - - name: total.pct - type: scaled_float - format: percentage - description: > - Total CPU usage. - # TODO: how to document cpu list? - #- name: core - # type: array - # description: > - # Dictionary with list of cpu and usage inside. - - - name: diskio - type: group - description: > - Disk I/O metrics. - fields: - - name: reads - type: scaled_float - description: > - Number of reads. - - name: writes - type: scaled_float - description: > - Number of writes. - - name: total - type: scaled_float - description: > - Number of reads and writes combined. - - - name: healthcheck - type: group - description: > - Docker container metrics. - fields: - - name: failingstreak - type: integer - description: > - concurent failed check - - name: status - type: keyword - description: > - Healthcheck status code - - name: event - type: group - description: > - event fields. - fields: - - name: end_date - type: date - description: > - Healthcheck end date - - name: start_date - type: date - description: > - Healthcheck start date - - name: output - type: keyword - description: > - Healthcheck output - - name: exit_code - type: integer - description: > - Healthcheck status code - - - name: image - type: group - description: > - Docker image metrics. - fields: - - name: id - type: group - description: > - The image layers identifier. - fields: - - name: current - type: keyword - description: > - Unique image identifier given upon its creation. - - name: parent - type: keyword - description: > - Identifier of the image, if it exists, from which the current image directly descends. - - name: created - type: date - description: > - Date and time when the image was created. - - name: size - type: group - description: > - Image size layers. - fields: - - name: virtual - type: long - description: > - Size of the image. - - name: regular - type: long - description: > - Total size of the all cached images associated to the current image. - - - name: labels - type: object - object_type: keyword - description: > - Image labels. - - - name: tags - type: array - description: > - Image tags. - - - name: info - type: group - description: > - beta[] - - Info metrics based on https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/display-system-wide-information. - fields: - - name: containers - type: group - description: > - Overall container stats. - fields: - - name: paused - type: long - description: > - Total number of paused containers. - - name: running - type: long - description: > - Total number of running containers. - - name: stopped - type: long - description: > - Total number of stopped containers. - - name: total - type: long - description: > - Total number of existing containers. - - name: id - type: keyword - description: > - Unique Docker host identifier. - - - name: images - type: long - description: > - Total number of existing images. - - - name: memory - type: group - description: > - Memory metrics. - fields: - - - name: fail.count - type: scaled_float - description: > - Fail counter. - - name: limit - type: long - format: bytes - description: > - Memory limit. - - name: rss - type: group - description: > - RSS memory stats. - fields: - - name: total - type: long - format: bytes - description: > - Total memory resident set size. - - name: pct - type: scaled_float - format: percentage - description: > - Memory resident set size percentage. - - name: usage - type: group - description: > - Usage memory stats. - fields: - - name: max - type: long - format: bytes - description: > - Max memory usage. - - name: pct - type: scaled_float - format: percentage - description: > - Memory usage percentage. - - name: total - type: long - format: bytes - description: > - Total memory usage. - - - name: network - type: group - description: > - Network metrics. - fields: - - - name: interface - type: keyword - description: > - Network interface name. - - name: in - type: group - description: > - Incoming network stats. - fields: - - name: bytes - type: long - format: bytes - description: > - Total number of incoming bytes. - - name: dropped - type: scaled_float - description: > - Total number of dropped incoming packets. - - name: errors - type: long - description: > - Total errors on incoming packets. - - name: packets - type: long - description: > - Total number of incoming packets. - - name: out - type: group - description: > - Outgoing network stats. - fields: - - name: bytes - type: long - format: bytes - description: > - Total number of outgoing bytes. - - name: dropped - type: scaled_float - description: > - Total number of dropped outgoing packets. - - name: errors - type: long - description: > - Total errors on outgoing packets. - - name: packets - type: long - description: > - Total number of outgoing packets. - -- key: dropwizard - title: "Dropwizard" - description: > - beta[] - - Stats collected from Dropwizard. - short_config: false - fields: - - name: dropwizard - type: group - description: > - fields: - -- key: elasticsearch - title: "Elasticsearch" - description: > - experimental[] - - Elasticsearch module - short_config: false - fields: - - name: elasticsearch - type: group - description: > - fields: - - name: cluster.name + - name: multifield_field + type: text + multi_fields: + - name: keyword type: keyword - description: > - Elasticsearch cluster name. - - name: node - type: group - description: > - node - fields: - - name: jvm.memory.heap_init.bytes - type: long - format: bytes - description: > - Heap init used by the JVM in bytes. - - name: jvm.version - type: keyword - description: > - JVM version. - - name: name - type: keyword - description: > - Node name. - - name: version - type: keyword - description: > - Node version. - - - name: node.stats - type: group - description: > - node_stats - fields: - - name: indices - type: group - description: > - Node indices stats - fields: - - name: docs.count - type: long - description: > - Total number of existing documents. - - name: docs.deleted - type: long - description: > - Total number of deleted documents. - - name: segments.count - type: long - description: > - Total number of segments. - - name: segments.memory.bytes - type: long - format: bytes - description: > - Total size of segments in bytes. - - name: store.size.bytes - type: long - description: > - Total size of the store in bytes. - - name: jvm.mem.pools - type: group - description: > - JVM memory pool stats - fields: - - name: old - type: group - description: > - Old memory pool stats. - fields: - - name: max.bytes - type: long - format: bytes - description: - Max bytes. - - name: peak.bytes - type: long - format: bytes - description: - Peak bytes. - - name: peak_max.bytes - type: long - format: bytes - description: - Peak max bytes. - - name: used.bytes - type: long - format: bytes - description: - Used bytes. - - name: young - type: group - description: > - Young memory pool stats. - fields: - - name: max.bytes - type: long - format: bytes - description: - Max bytes. - - name: peak.bytes - type: long - format: bytes - description: - Peak bytes. - - name: peak_max.bytes - type: long - format: bytes - description: - Peak max bytes. - - name: used.bytes - type: long - format: bytes - description: - Used bytes. - - name: survivor - type: group - description: > - Survivor memory pool stats. - fields: - - name: max.bytes - type: long - format: bytes - description: - Max bytes. - - name: peak.bytes - type: long - format: bytes - description: - Peak bytes. - - name: peak_max.bytes - type: long - format: bytes - description: - Peak max bytes. - - name: used.bytes - type: long - format: bytes - description: - Used bytes. - - - name: jvm.gc.collectors - type: group - description: > - GC collector stats. - fields: - - name: old.collection - type: group - description: > - Old collection gc. - fields: - - name: count - type: long - description: > - - name: ms - type: long - description: > - - name: young.collection - type: group - description: > - Young collection gc. - fields: - - name: count - type: long - description: > - - name: ms - type: long - description: > - -- key: golang - title: "Golang" - description: > - Golang module - short_config: false - fields: - - name: golang - type: group - description: > - fields: - - name: expvar - type: group - description: > - expvar - fields: - - name: cmdline - type: keyword - description: > - The cmdline of this golang program start with. - - name: heap - type: group - description: > - The golang program heap information exposed by expvar. - fields: - - name: cmdline - type: keyword - description: > - The cmdline of this golang program start with. - - - name: gc - type: group - description: > - Garbage collector summary. - fields: - - name: total_pause - type: group - description: > - Total GC pause duration over lifetime of process. - fields: - - name: ns - type: long - description: > - Duration in Ns. - - name: total_count - type: long - description: > - Total number of GC was happened. - - name: next_gc_limit - type: long - format: bytes - description: > - Next collection will happen when HeapAlloc > this amount. - - name: cpu_fraction - type: long - description: > - Fraction of CPU time used by GC. - - name: pause - type: group - description: > - Last GC pause durations during the monitoring period. - fields: - - name: count - type: long - description: > - Count of GC pause duration during this collect period. - - name: sum - type: group - description: > - Total GC pause duration during this collect period. - fields: - - name: ns - type: long - description: > - Duration in Ns. - - name: max - type: group - description: > - Max GC pause duration during this collect period. - fields: - - name: ns - type: long - description: > - Duration in Ns. - - name: avg - type: group - description: > - Average GC pause duration during this collect period. - fields: - - name: ns - type: long - description: > - Duration in Ns. - - - name: system - type: group - description: > - Heap summary,which bytes was obtained from system. - fields: - - name: total - type: long - format: bytes - description: > - Total bytes obtained from system (sum of XxxSys below). - - name: optained - type: long - format: bytes - description: > - Via HeapSys, bytes obtained from system. heap_sys = heap_idle + heap_inuse. - - name: stack - type: long - format: bytes - description: > - Bytes used by stack allocator, and these bytes was obtained from system. - - name: released - type: long - format: bytes - description: > - Bytes released to the OS. - - - name: allocations - type: group - description: > - Heap allocations summary. - fields: - - name: mallocs - type: long - description: > - Number of mallocs. - - name: frees - type: long - description: > - Number of frees. - - name: objects - type: long - description: > - Total number of allocated objects. - - name: total - type: long - format: bytes - description: > - Bytes allocated (even if freed) throughout the lifetime. - - name: allocated - type: long - format: bytes - description: > - Bytes allocated and not yet freed (same as Alloc above). - - name: idle - type: long - format: bytes - description: > - Bytes in idle spans. - - name: active - type: long - format: bytes - description: > - Bytes in non-idle span. -- key: graphite - title: "graphite" - description: > - []experimental - - graphite Module - fields: - - name: graphite - type: group - description: > - fields: - - name: server - type: group - description: > - server - fields: - - name: example - type: keyword - description: > - Example field - -- key: haproxy - title: "HAProxy" - description: > - HAProxy Module - short_config: false - fields: - - name: haproxy - type: group - description: > - HAProxy metrics. - fields: - - name: info - type: group - description: > - General information about HAProxy processes. - fields: - - name: processes - type: long - description: > - Number of processes. - - - name: process_num - type: long - description: > - Process number. - - - name: pid - type: long - description: > - Process ID. - - - name: run_queue - type: long - description: > - - - name: tasks - type: long - description: > - - - name: uptime.sec - type: long - description: > - Current uptime in seconds. - - - name: memory.max.bytes - type: long - format: bytes - description: > - Maximum amount of memory usage in bytes (the 'Memmax_MB' value converted to bytes). - - - name: ulimit_n - type: long - description: > - Maximum number of open files for the process. - - - name: compress - type: group - description: > - - fields: - - name: bps - type: group - description: > - - fields: - - name: in - type: long - description: > - - - name: out - type: long - description: > - - - name: rate_limit - type: long - description: > - - - name: connection - type: group - description: > - - fields: - - name: rate - type: group - description: > - - fields: - - name: value - type: long - description: > - - - name: limit - type: long - description: > - - - name: max - type: long - description: > - - - name: current - type: long - description: > - Current connections. - - - name: total - type: long - description: > - Total connections. - - - name: ssl.current - type: long - description: > - Current SSL connections. - - - name: ssl.total - type: long - description: > - Total SSL connections. - - - name: ssl.max - type: long - description: > - Maximum SSL connections. - - - name: max - type: long - description: > - Maximum connections. - - - name: hard_max - type: long - description: > - - - name: requests.total - type: long - description: > - - - name: sockets.max - type: long - description: > - - - name: requests.max - type: long - description: > - - - name: pipes - type: group - description: > - fields: - - name: used - type: integer - description: > - - - name: free - type: integer - description: > - - - name: max - type: integer - description: > - - - name: session - type: group - description: - fields: - - name: rate.value - type: integer - description: > - - - name: rate.limit - type: integer - description: > - - - name: rate.max - type: integer - description: > - - - - name: ssl - type: group - description: - fields: - - name: rate.value - type: integer - description: - - - name: rate.limit - type: integer - description: - - - name: rate.max - type: integer - description: - - - name: frontend - type: group - description: - fields: - - name: key_rate.value - type: integer - description: - - - name: key_rate.max - type: integer - description: - - - name: session_reuse.pct - type: scaled_float - format: percent - description: - - - name: backend - type: group - description: - fields: - - name: key_rate.value - type: integer - description: - - - name: key_rate.max - type: integer - description: MaxConnRate - - name: cached_lookups - type: long - description: - - name: cache_misses - type: long - description: - - - - name: zlib_mem_usage - type: group - description: > - - fields: - - name: value - type: integer - description: > - - - name: max - type: integer - description: > - - - name: idle.pct - type: scaled_float - format: percent - description: > - - - name: stat - type: group - description: > - Stats collected from HAProxy processes. - fields: - - - name: status - type: keyword - description: > - Status (UP, DOWN, NOLB, MAINT, or MAINT(via)...). - - - name: weight - type: long - description: > - Total weight (for backends), or server weight (for servers). - - - name: downtime - type: long - description: > - Total downtime (in seconds). For backends, this value is the downtime - for the whole backend, not the sum of the downtime for the servers. - - - name: component_type - type: integer - description: > - Component type (0=frontend, 1=backend, 2=server, or 3=socket/listener). - - - name: process_id - type: integer - description: > - Process ID (0 for first instance, 1 for second, and so on). - - - name: service_name - type: keyword - description: > - Service name (FRONTEND for frontend, BACKEND for backend, or any name for server/listener). - - - name: in.bytes - type: long - format: bytes - description: > - Bytes in. - - - name: out.bytes - type: long - format: bytes - description: > - Bytes out. - - - name: last_change - type: integer - description: > - Number of seconds since the last UP->DOWN or DOWN->UP transition. - - - name: throttle.pct - type: scaled_float - format: percentage - description: > - Current throttle percentage for the server when slowstart - is active, or no value if slowstart is inactive. - - - name: selected.total - type: long - description: > - Total number of times a server was selected, either for new - sessions, or when re-dispatching. For servers, this field reports the - the number of times the server was selected. - - - name: tracked.id - type: long - description: > - ID of the proxy/server if tracking is enabled. - - - name: connection - type: group - fields: - - - name: total - type: long - description: > - Cumulative number of connections. - - - name: retried - type: long - description: > - Number of times a connection to a server was retried. - - - name: time.avg - type: long - description: > - Average connect time in ms over the last 1024 requests. - - - name: request - type: group - fields: - - - name: denied - type: long - description: > - Requests denied because of security concerns. - - * For TCP this is because of a matched tcp-request content rule. - * For HTTP this is because of a matched http-request or tarpit rule. - - - name: queued.current - type: long - description: > - Current queued requests. For backends, this field reports the number - of requests queued without a server assigned. - - - name: queued.max - type: long - description: > - Maximum value of queued.current. - - - name: errors - type: long - description: > - Request errors. Some of the possible causes are: - - * early termination from the client, before the request has been sent - * read error from the client - * client timeout - * client closed connection - * various bad requests from the client. - * request was tarpitted. - - - name: redispatched - type: long - description: > - Number of times a request was redispatched to another server. For - servers, this field reports the number of times the server was - switched away from. - - - name: connection.errors - type: long - description: > - Number of requests that encountered an error trying to - connect to a server. For backends, this field reports the sum of - the stat for all backend servers, plus any connection errors not - associated with a particular server (such as the backend having no - active servers). - - - name: rate - type: group - description: > - fields: - - name: value - type: long - description: > - Number of HTTP requests per second over the last elapsed second. - - name: max - type: long - description: > - Maximum number of HTTP requests per second. - - - name: total - type: long - description: > - Total number of HTTP requests received. - - - - name: response - type: group - fields: - - - name: errors - type: long - description: > - Number of response errors. This value includes the number of data - transfers aborted by the server (haproxy.stat.server.aborted). - Some other errors are: - - * write errors on the client socket (won't be counted for the server stat) - * failure applying filters to the response - - - - name: time.avg - type: long - description: > - Average response time in ms over the last 1024 requests (0 for TCP). - - - name: denied - type: integer - description: > - Responses denied because of security concerns. For HTTP this is - because of a matched http-request rule, or "option checkcache". - - - name: http - type: group - description: > - - fields: - - name: 1xx - type: long - description: > - HTTP responses with 1xx code. - - - name: 2xx - type: long - description: > - HTTP responses with 2xx code. - - - name: 3xx - type: long - description: > - HTTP responses with 3xx code. - - - name: 4xx - type: long - description: > - HTTP responses with 4xx code. - - - name: 5xx - type: long - description: > - HTTP responses with 5xx code. - - - name: other - type: long - description: > - HTTP responses with other codes (protocol error). - - - - name: session - type: group - fields: - - - name: current - type: long - description: > - Number of current sessions. - - - name: max - type: long - description: > - Maximum number of sessions. - - - name: limit - type: long - description: > - Configured session limit. - - - name: rate - type: group - fields: - - name: value - type: integer - description: > - Number of sessions per second over the last elapsed second. - - - name: limit - type: integer - description: > - Configured limit on new sessions per second. - - - name: max - type: integer - description: > - Maximum number of new sessions per second. - - - - name: check - type: group - description: > - - fields: - - name: status - type: keyword - description: > - Status of the last health check. One of: - - UNK -> unknown - INI -> initializing - SOCKERR -> socket error - L4OK -> check passed on layer 4, no upper layers testing enabled - L4TOUT -> layer 1-4 timeout - L4CON -> layer 1-4 connection problem, for example - "Connection refused" (tcp rst) or "No route to host" (icmp) - L6OK -> check passed on layer 6 - L6TOUT -> layer 6 (SSL) timeout - L6RSP -> layer 6 invalid response - protocol error - L7OK -> check passed on layer 7 - L7OKC -> check conditionally passed on layer 7, for example 404 with - disable-on-404 - L7TOUT -> layer 7 (HTTP/SMTP) timeout - L7RSP -> layer 7 invalid response - protocol error - L7STS -> layer 7 response error, for example HTTP 5xx - - - name: code - type: long - description: > - Layer 5-7 code, if available. - - - name: duration - type: long - description: > - Time in ms that it took to finish the last health check. - - - name: health.last - type: keyword - description: > - The result of the last health check. - - - name: health.fail - type: long - description: > - Number of failed checks. - - - name: agent.last - type: integer - description: > - - - name: failed - type: long - description: > - Number of checks that failed while the server was up. - - - name: down - type: long - description: > - Number of UP->DOWN transitions. For backends, this value is the - number of transitions to the whole backend being down, rather than - the sum of the transitions for each server. - - - name: client.aborted - type: integer - description: > - Number of data transfers aborted by the client. - - - - name: server - type: group - description: > - fields: - - - name: id - type: integer - description: > - Server ID (unique inside a proxy). - - - name: aborted - type: integer - description: > - Number of data transfers aborted by the server. This value is - included in haproxy.stat.response.errors. - - - name: active - type: integer - description: > - Number of backend servers that are active, meaning that they are - healthy and can receive requests from the load balancer. - - - name: backup - type: integer - description: > - Number of backend servers that are backup servers. - - - - name: compressor - type: group - description: > - - fields: - - name: in.bytes - type: long - format: bytes - description: > - Number of HTTP response bytes fed to the compressor. - - - name: out.bytes - type: integer - format: bytes - description: > - Number of HTTP response bytes emitted by the compressor. - - - name: bypassed.bytes - type: long - format: bytes - description: > - Number of bytes that bypassed the HTTP compressor (CPU/BW limit). - - - name: response.bytes - type: long - format: bytes - description: > - Number of HTTP responses that were compressed. - - - name: proxy - type: group - description: > - - fields: - - name: id - type: integer - description: > - Unique proxy ID. - - - name: name - type: keyword - description: > - Proxy name. - - - - name: queue - type: group - description: > - - fields: - - name: limit - type: integer - description: > - Configured queue limit (maxqueue) for the server, or nothing if the - value of maxqueue is 0 (meaning no limit). - - - name: time.avg - type: integer - description: > - The average queue time in ms over the last 1024 requests. - - - - - -- key: http - title: "HTTP" - description: > - HTTP module - fields: - - name: http - type: group - description: > - fields: - - name: request - type: group - description: > - HTTP request information - fields: - - name: header - type: object - description: > - The HTTP headers sent - - name: method - type: keyword - description: > - The HTTP method used - - name: body - type: keyword - description: > - The HTTP payload sent - - name: response - type: group - description: > - HTTP response information - fields: - - name: header - type: object - description: > - The HTTP headers received - - name: status_code - type: keyword - description: > - The HTTP status code - - name: body - type: keyword - description: > - The HTTP payload received - - name: json - type: group - description: > - json metricset - fields: - - - name: server - type: group - description: > - server - fields: - -- key: jolokia - title: "Jolokia" - description: > - beta[] - - Jolokia module - short_config: false - fields: - - name: jolokia - type: group - description: > - jolokia contains metrics exposed via jolokia agent - fields: - -- key: kafka - title: "Kafka" - description: > - Kafka module - - beta[] - short_config: false - fields: - - name: kafka - type: group - description: > - fields: - - name: consumergroup - type: group - description: > - consumergroup - fields: - - name: broker - type: group - description: > - Broker Consumer Group Information have been read from (Broker handling - the consumer group). - fields: - - name: id - type: long - description: > - Broker id - - - name: address - type: keyword - description: > - Broker address - - - name: id - type: keyword - description: Consumer Group ID - - - name: topic - type: keyword - description: Topic name - - - name: partition - type: long - description: Partition ID - - - name: offset - type: long - description: consumer offset into partition being read - - - name: meta - type: text - description: custom consumer meta data string - - - name: error.code - type: long - description: > - kafka consumer/partition error code. - - - name: client - type: group - description: > - Assigned client reading events from partition - fields: - - name: id - type: keyword - description: Client ID (kafka setting client.id) - - - name: host - type: keyword - description: Client host - - - name: member_id - type: keyword - description: internal consumer group member ID - - - name: partition - type: group - description: > - partition - fields: - - name: offset - type: group - description: > - Available offsets of the given partition. - fields: - - name: newest - type: long - description: > - Newest offset of the partition. - - name: oldest - type: long - description: > - Oldest offset of the partition. - - - name: partition - type: group - description: > - Partition data. - fields: - - name: id - type: long - description: > - Partition id. - - - name: leader - type: long - description: > - Leader id (broker). - - name: isr - type: array - description: > - List of isr ids. - - name: replica - type: long - description: > - Replica id (broker). - - - name: insync_replica - type: boolean - description: > - Indicates if replica is included in the in-sync replicate set (ISR). - - - name: error.code - type: long - description: > - Error code from fetching partition. - - - name: topic.error.code - type: long - description: > - topic error code. - - name: topic.name - type: keyword - description: > - Topic name - - - name: broker.id - type: long - description: > - Broker id - - name: broker.address - type: keyword - description: > - Broker address - - - -- key: kibana - title: "Kibana" - description: > - experimental[] - - Kibana module - short_config: false - fields: - - name: kibana - type: group - description: > - fields: - - name: status - type: group - description: > - Status fields - fields: - - name: name - type: keyword - description: > - Kibana instance name. - - name: uuid - type: keyword - description: > - Kibana instance uuid. - - name: version.number - type: keyword - description: > - Kibana version number. - - name: status.overall.state - type: keyword - description: > - Kibana overall state. - - name: metrics - type: group - description: > - Metrics fields - fields: - - name: concurrent_connections - type: long - description: > - Current concurrent connections. - - name: requests - type: group - description: > - Request statistics. - fields: - - name: disconnects - type: long - description: > - Total number of disconnected connections. - - name: total - type: long - description: > - Total number of connections. - -- key: kubernetes - title: "Kubernetes" - description: > - beta[] - - Kubernetes metrics - short_config: false - fields: - - name: kubernetes - type: group - description: > - Information and statistics of pods managed by kubernetes. - fields: - - name: container - type: group - description: > - kubernetes container metrics - fields: - - name: start_time - type: date - description: > - Start time - - name: cpu - type: group - description: > - CPU usage metrics - fields: - - name: usage - type: group - fields: - - name: core - type: group - fields: - - name: ns - type: long - description: > - Container CPU Core usage nanoseconds - - name: nanocores - type: long - description: > - CPU used nanocores - - name: logs - type: group - description: > - Logs info - fields: - - name: available - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Logs available capacity in bytes - - name: capacity - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Logs total capacity in bytes - - name: used - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Logs used capacity in bytes - - name: inodes - type: group - fields: - - name: count - type: long - description: > - Total available inodes - - name: free - type: long - description: > - Total free inodes - - name: used - type: long - description: > - Total used inodes - - name: memory - type: group - fields: - - name: available - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Total available memory - - name: usage - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Total memory usage - - name: rss - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - RSS memory usage - - name: workingset - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Working set memory usage - - name: pagefaults - type: long - description: > - Number of page faults - - name: majorpagefaults - type: long - description: > - Number of major page faults - - name: rootfs - type: group - fields: - - name: capacity - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Root filesystem total capacity in bytes - - name: available - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Root filesystem total available in bytes - - name: used - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Root filesystem total used in bytes - - name: inodes - type: group - fields: - - name: used - type: long - description: > - Used inodes - - - name: event - type: group - description: > - The Kubernetes events metricset collects events that are generated by objects running - inside of Kubernetes - fields: - - name: count - type: long - description: > - Count field records the number of times the particular event has occured - fields: - - name: timestamp - type: group - fields: - - name: first_occurrence - type: date - description: > - Timestamp of first occurrence of event - - name: last_occurrence - type: date - description: > - Timestamp of last occurrence of event - - name: message - type: keyword - description: > - Message recorded for the given event - - name: reason - type: keyword - description: > - Reason recorded for the given event - - name: type - type: keyword - description: > - Type of the given event - - name: metadata - type: group - description: > - Metadata associated with the given event - fields: - - name: timestamp - type: group - fields: - - name: created - type: date - description: > - Timestamp of creation of the given event - - name: name - type: keyword - description: > - Name of the event - - name: namespace - type: keyword - description: > - Namespace in which event was generated - - name: resource_version - type: keyword - description: > - Version of the event resource - - name: uid - type: keyword - description: > - Unique identifier to the event object - - name: self_link - type: keyword - description: > - URL representing the event - - name: involved_object - type: group - description: > - Metadata associated with the given involved object - fields: - - name: api_version - type: keyword - description: > - API version of the object - - name: kind - type: keyword - description: > - API kind of the object - - name: name - type: keyword - description: > - name of the object - - name: resource_version - type: keyword - description: > - resource version of the object - - name: uid - type: keyword - description: > - UUID version of the object - - - name: node - type: group - description: > - kubernetes node metrics - fields: - - name: name - type: keyword - description: > - Node name - - name: start_time - type: date - description: > - Start time - - name: cpu - type: group - description: > - CPU usage metrics - fields: - - name: usage - type: group - fields: - - name: core - type: group - fields: - - name: ns - type: long - description: > - Node CPU Core usage nanoseconds - - name: nanocores - type: long - description: > - CPU used nanocores - - name: memory - type: group - fields: - - name: available - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Total available memory - - name: usage - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Total memory usage - - name: rss - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - RSS memory usage - - name: workingset - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Working set memory usage - - name: pagefaults - type: long - description: > - Number of page faults - - name: majorpagefaults - type: long - description: > - Number of major page faults - - name: network - type: group - fields: - - name: rx - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Received bytes - - name: errors - type: long - description: > - Rx errors - - name: tx - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Transmitted bytes - - name: errors - type: long - description: > - Tx errors - - name: fs - type: group - fields: - - name: capacity - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Filesystem total capacity in bytes - - name: available - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Filesystem total available in bytes - - name: used - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Filesystem total used in bytes - - name: inodes - type: group - fields: - - name: used - type: long - description: > - Number of used inodes - - name: count - type: long - description: > - Number of inodes - - name: free - type: long - description: > - Number of free inodes - - name: runtime - type: group - fields: - - name: imagefs - type: group - fields: - - name: capacity - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Image filesystem total capacity in bytes - - name: available - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Image filesystem total available in bytes - - name: used - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Image filesystem total used in bytes - - - name: pod - type: group - description: > - kubernetes pod metrics - fields: - - name: start_time - type: date - description: > - Start time - - name: network - type: group - fields: - - name: rx - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Received bytes - - name: errors - type: long - description: > - Rx errors - - name: tx - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Transmitted bytes - - name: errors - type: long - description: > - Tx errors - - - name: container - type: group - description: > - kubernetes container metrics - fields: - - name: id - type: keyword - description: - Container id - - name: status - type: group - fields: - - name: phase - type: keyword - description: > - Container phase (running, waiting, terminated) - - name: ready - type: boolean - description: > - Container ready status - - name: restarts - type: integer - description: > - Container restarts count - - name: cpu - type: group - fields: - - name: limit.nanocores - type: long - description: > - Container CPU nanocores limit - - name: request.nanocores - type: long - description: > - Container CPU requested nanocores - - name: memory - type: group - fields: - - name: limit.bytes - type: long - format: bytes - description: > - Container memory limit in bytes - - name: request.bytes - type: long - format: bytes - description: > - Container requested memory in bytes - - - name: deployment - type: group - description: > - kubernetes deployment metrics - fields: - - name: name - type: keyword - description: > - Kubernetes deployment name - - name: paused - type: boolean - description: > - Kubernetes deployment paused status - - name: replicas - type: group - description: > - Kubernetes deployment replicas info - fields: - - name: desired - type: integer - description: > - Deployment number of desired replicas (spec) - - name: available - type: integer - description: > - Deployment available replicas - - name: unavailable - type: integer - description: > - Deployment unavailable replicas - - name: updated - type: integer - description: > - Deployment updated replicas - - - name: node - type: group - description: > - kubernetes node metrics - fields: - - name: status - type: group - fields: - - name: ready - type: keyword - description: > - Node ready status (true, false or unknown) - - name: unschedulable - type: boolean - description: > - Node unschedulable status - - name: cpu - type: group - fields: - - name: allocatable.cores - type: long - description: > - Node CPU allocatable cores - - name: capacity.cores - type: long - description: > - Node CPU capacity cores - - name: memory - type: group - fields: - - name: allocatable.bytes - type: long - format: bytes - description: > - Node allocatable memory in bytes - - name: capacity.bytes - type: long - format: bytes - description: > - Node memory capacity in bytes - - name: pod - type: group - fields: - - name: allocatable.total - type: long - description: > - Node allocatable pods - - name: capacity.total - type: long - description: > - Node pod capacity - - - name: pod - type: group - description: > - kubernetes pod metrics - fields: - - name: ip - type: ip - description: > - Kubernetes pod IP - - name: host_ip - type: ip - description: > - Kubernetes pod host IP - - name: status - type: group - description: > - Kubernetes pod status metrics - fields: - - name: phase - type: keyword - description: > - Kubernetes pod phase (Running, Pending...) - - name: ready - type: keyword - description: > - Kubernetes pod ready status (true, false or unknown) - - name: scheduled - type: keyword - description: > - Kubernetes pod scheduled status (true, false, unknown) - - - name: replicaset - type: group - description: > - kubernetes replica set metrics - fields: - - name: name - type: keyword - description: > - Kubernetes replica set name - - name: replicas - type: group - description: > - Kubernetes replica set paused status - fields: - - name: available - type: long - description: > - The number of replicas per ReplicaSet - - name: desired - type: long - description: > - The number of replicas per ReplicaSet - - name: ready - type: long - description: > - The number of ready replicas per ReplicaSet - - name: observed - type: long - description: > - The generation observed by the ReplicaSet controller - - name: labeled - type: long - description: > - The number of fully labeled replicas per ReplicaSet - - - name: system - type: group - description: > - kubernetes system containers metrics - fields: - - name: container - type: keyword - description: > - Container name - - name: start_time - type: date - description: > - Start time - - name: cpu - type: group - description: > - CPU usage metrics - fields: - - name: usage - type: group - fields: - - name: core - type: group - fields: - - name: ns - type: long - description: > - CPU Core usage nanoseconds - - name: nanocores - type: long - description: > - CPU used nanocores - - name: memory - type: group - fields: - - name: usage - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Total memory usage - - name: rss - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - RSS memory usage - - name: workingset - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Working set memory usage - - name: pagefaults - type: long - description: > - Number of page faults - - name: majorpagefaults - type: long - description: > - Number of major page faults - - - name: volume - type: group - description: > - kubernetes volume metrics - fields: - - name: name - type: keyword - description: > - Volume name - - name: fs - type: group - fields: - - name: capacity - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Filesystem total capacity in bytes - - name: available - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Filesystem total available in bytes - - name: used - type: group - fields: - - name: bytes - type: long - format: bytes - description: > - Filesystem total used in bytes - - name: inodes - type: group - fields: - - name: used - type: long - description: > - Used inodes - - name: free - type: long - description: > - Free inodes - - name: count - type: long - description: > - Total inodes - -- key: memcached - title: "Memcached" - description: > - beta[] - - Memcached module - short_config: false - fields: - - name: memcached - type: group - description: > - fields: - - name: stats - type: group - description: > - stats - fields: - - name: pid - type: long - description: > - Current process ID of the Memcached task. - - - name: uptime.sec - type: long - description: > - Memcached server uptime. - - - name: threads - type: long - description: > - Number of threads used by the current Memcached server process. - - - name: connections.current - type: long - description: > - Number of open connections to this Memcached server, should be the same - value on all servers during normal operation. - - - name: connections.total - type: long - description: > - Numer of successful connect attempts to this server since it has been started. - - - name: get.hits - type: long - description: > - Number of successful "get" commands (cache hits) since startup, divide them - by the "cmd_get" value to get the cache hitrate. - - - name: get.misses - type: long - description: > - Number of failed "get" requests because nothing was cached for this key - or the cached value was too old. - - - name: cmd.get - type: long - description: > - Number of "get" commands received since server startup not counting if they - were successful or not. - - - name: cmd.set - type: long - description: > - Number of "set" commands serviced since startup. - - - name: read.bytes - type: long - formate: bytes - description: > - Total number of bytes received from the network by this server. - - - name: written.bytes - type: long - formate: bytes - description: > - Total number of bytes send to the network by this server. - - - name: items.current - type: long - description: > - Number of items currently in this server's cache. - - - name: items.total - type: long - formate: bytes - description: > - Number of items stored ever stored on this server. This is no "maximum item - count" value but a counted increased by every new item stored in the cache. - - - name: evictions - type: long - formate: bytes - description: > - Number of objects removed from the cache to free up memory for new items - because Memcached reached it's maximum memory setting (limit_maxbytes). - -- key: mongodb - title: "MongoDB" - description: > - Metrics collected from MongoDB servers. - short_config: false - fields: - - name: mongodb - type: group - description: > - MongoDB metrics. - fields: - - name: dbstats - type: group - description: > - dbstats provides an overview of a particular mongo database. This document - is most concerned with data volumes of a database. - fields: - - name: avg_obj_size.bytes - type: long - format: bytes - - - name: collections - type: integer - - - name: data_size.bytes - type: long - format: bytes - - - name: db - type: keyword - - - name: file_size.bytes - type: long - format: bytes - - - name: index_size.bytes - type: long - format: bytes - - - name: indexes - type: long - - - name: num_extents - type: long - - - name: objects - type: long - - - name: storage_size.bytes - type: long - format: bytes - - - name: ns_size_mb.mb - type: long - - - name: data_file_version - type: group - fields: - - name: major - type: long - - - name: minor - type: long - - - name: extent_free_list - type: group - fields: - - name: num - type: long - - - name: size.bytes - type: long - format: bytes - - - name: status - type: group - description: > - MongoDB server status metrics. - fields: - - name: version - type: keyword - description: > - Instance version. - - name: uptime.ms - type: long - description: > - Instance uptime in milliseconds. - - name: local_time - type: date - description: > - Local time as reported by the MongoDB instance. - - - name: asserts.regular - type: long - description: > - Number of regular assertions produced by the server. - - name: asserts.warning - type: long - description: > - Number of warning assertions produced by the server. - - name: asserts.msg - type: long - description: > - Number of msg assertions produced by the server. - - name: asserts.user - type: long - description: > - Number of user assertions produced by the server. - - name: asserts.rollovers - type: long - description: > - Number of rollovers assertions produced by the server. - - - name: background_flushing - type: group - description: > - Data about the process MongoDB uses to write data to disk. This data is - only available for instances that use the MMAPv1 storage engine. - fields: - - name: flushes - type: long - description: > - A counter that collects the number of times the database has - flushed all writes to disk. - - name: total.ms - type: long - description: > - The total number of milliseconds (ms) that the mongod processes have - spent writing (i.e. flushing) data to disk. Because this is an - absolute value, consider the value of `flushes` and `average_ms` to - provide better context for this datum. - - name: average.ms - type: long - description: > - The average time spent flushing to disk per flush event. - - name: last.ms - type: long - description: > - The amount of time, in milliseconds, that the last flush operation - took to complete. - - name: last_finished - type: date - description: > - A timestamp of the last completed flush operation. - - - name: connections - type: group - description: > - Data regarding the current status of incoming connections and - availability of the database server. - fields: - - name: current - type: long - description: > - The number of connections to the database server from clients. This - number includes the current shell session. Consider the value of - `available` to add more context to this datum. - - name: available - type: long - description: > - The number of unused available incoming connections the database - can provide. - - name: total_created - type: long - description: > - A count of all incoming connections created to the server. This - number includes connections that have since closed. - - - name: journaling - type: group - description: > - Data about the journaling-related operations and performance. Journaling - information only appears for mongod instances that use the MMAPv1 - storage engine and have journaling enabled. - fields: - - name: commits - type: long - description: > - The number of transactions written to the journal during the last - journal group commit interval. - - name: journaled.mb - type: long - description: > - The amount of data in megabytes (MB) written to journal during the - last journal group commit interval. - - name: write_to_data_files.mb - type: long - description: > - The amount of data in megabytes (MB) written from journal to the - data files during the last journal group commit interval. - - name: compression - type: long - description: > - The compression ratio of the data written to the journal. - - name: commits_in_write_lock - type: long - description: > - Count of the commits that occurred while a write lock was held. - Commits in a write lock indicate a MongoDB node under a heavy write - load and call for further diagnosis. - - name: early_commits - type: long - description: > - The number of times MongoDB requested a commit before the scheduled - journal group commit interval. - - name: times - type: group - description: > - Information about the performance of the mongod instance during the - various phases of journaling in the last journal group commit - interval. - fields: - - name: dt.ms - type: long - description: > - The amount of time over which MongoDB collected the times data. - Use this field to provide context to the other times field values. - - name: prep_log_buffer.ms - type: long - description: > - The amount of time spent preparing to write to the journal. - Smaller values indicate better journal performance. - - name: write_to_journal.ms - type: long - description: > - The amount of time spent actually writing to the journal. File - system speeds and device interfaces can affect performance. - - name: write_to_data_files.ms - type: long - description: > - The amount of time spent writing to data files after journaling. - File system speeds and device interfaces can affect performance. - - name: remap_private_view.ms - type: long - description: > - The amount of time spent remapping copy-on-write memory mapped - views. Smaller values indicate better journal performance. - - name: commits.ms - type: long - description: > - The amount of time spent for commits. - - name: commits_in_write_lock.ms - type: long - description: > - The amount of time spent for commits that occurred while a write - lock was held. - - - name: extra_info - type: group - description: > - Platform specific data. - fields: - - name: heap_usage.bytes - type: long - format: bytes - description: > - The total size in bytes of heap space used by the database process. - Only available on Unix/Linux. - - name: page_faults - type: long - description: > - The total number of page faults that require disk operations. Page - faults refer to operations that require the database server to - access data that isn't available in active memory. - - - name: network - type: group - description: > - Platform specific data. - fields: - - name: in.bytes - type: long - format: bytes - description: > - The amount of network traffic, in bytes, received by this database. - - name: out.bytes - type: long - format: bytes - description: > - The amount of network traffic, in bytes, sent from this database. - - name: requests - type: long - description: > - The total number of requests received by the server. - - - name: opcounters - type: group - description: > - An overview of database operations by type. - fields: - - name: insert - type: long - description: > - The total number of insert operations received since the mongod - instance last started. - - name: query - type: long - description: > - The total number of queries received since the mongod instance last - started. - - name: update - type: long - description: > - The total number of update operations received since the mongod - instance last started. - - name: delete - type: long - description: > - The total number of delete operations received since the mongod - instance last started. - - name: getmore - type: long - description: > - The total number of getmore operations received since the mongod - instance last started. - - name: command - type: long - description: > - The total number of commands issued to the database since the mongod - instance last started. - - - name: opcounters_replicated - type: group - description: > - An overview of database replication operations by type. - fields: - - name: insert - type: long - description: > - The total number of replicated insert operations received since the - mongod instance last started. - - name: query - type: long - description: > - The total number of replicated queries received since the mongod - instance last started. - - name: update - type: long - description: > - The total number of replicated update operations received since the - mongod instance last started. - - name: delete - type: long - description: > - The total number of replicated delete operations received since the - mongod instance last started. - - name: getmore - type: long - description: > - The total number of replicated getmore operations received since the - mongod instance last started. - - name: command - type: long - description: > - The total number of replicated commands issued to the database since - the mongod instance last started. - - - name: memory - type: group - description: > - Data about the current memory usage of the mongod server. - fields: - - name: bits - type: long - description: > - Either 64 or 32, depending on which target architecture was specified - during the mongod compilation process. - - name: resident.mb - type: long - description: > - The amount of RAM, in megabytes (MB), currently used by the database - process. - - name: virtual.mb - type: long - description: > - The amount, in megabytes (MB), of virtual memory used by the mongod - process. - - name: mapped.mb - type: long - description: > - The amount of mapped memory, in megabytes (MB), used by the database. - Because MongoDB uses memory-mapped files, this value is likely to be - to be roughly equivalent to the total size of your database or - databases. - - name: mapped_with_journal.mb - type: long - description: > - The amount of mapped memory, in megabytes (MB), including the memory - used for journaling. - - name: write_backs_queued - type: boolean - description: > - True when there are operations from a mongos instance queued for retrying. - - name: storage_engine.name - type: keyword - description: > - A string that represents the name of the current storage engine. - - - name: wired_tiger - type: group - description: > - Statistics about the WiredTiger storage engine. - fields: - - name: concurrent_transactions - type: group - description: > - Statistics about the transactions currently in progress. - fields: - - name: write.out - type: long - description: > - Number of concurrent write transaction in progress. - - name: write.available - type: long - description: > - Number of concurrent write tickets available. - - name: write.total_tickets - type: long - description: > - Number of total write tickets. - - name: read.out - type: long - description: > - Number of concurrent read transaction in progress. - - name: read.available - type: long - description: > - Number of concurrent read tickets available. - - name: read.total_tickets - type: long - description: > - Number of total read tickets. - - name: cache - type: group - description: > - Statistics about the cache and page evictions from the cache. - fields: - - name: maximum.bytes - type: long - format: bytes - description: > - Maximum cache size. - - name: used.bytes - type: long - format: bytes - description: > - Size in byte of the data currently in cache. - - name: dirty.bytes - type: long - format: bytes - description: > - Size in bytes of the dirty data in the cache. - - name: pages.read - type: long - description: > - Number of pages read into the cache. - - name: pages.write - type: long - description: > - Number of pages written from the cache. - - name: pages.evicted - type: long - description: > - Number of pages evicted from the cache. - - name: log - type: group - description: > - Statistics about the write ahead log used by WiredTiger. - fields: - - name: size.bytes - type: long - format: bytes - description: > - Total log size in bytes. - - name: write.bytes - type: long - format: bytes - description: > - Number of bytes written into the log. - - name: max_file_size.bytes - type: long - format: bytes - description: > - Maximum file size. - - name: flushes - type: long - description: > - Number of flush operations. - - name: writes - type: long - description: > - Number of write operations. - - name: scans - type: long - description: > - Number of scan operations. - - name: syncs - type: long - description: > - Number of sync operations. - -- key: mysql - title: "MySQL" - description: > - MySQL server status metrics collected from MySQL. - short_config: false - fields: - - name: mysql - type: group - description: > - `mysql` contains the metrics that were obtained from MySQL - query. - fields: - - name: status - type: group - description: > - `status` contains the metrics that were obtained by the status SQL query. - fields: - - name: aborted - type: group - description: > - Aborted status fields. - fields: - - name: clients - type: long - description: > - The number of connections that were aborted because the client died without closing the connection properly. - - - name: connects - type: long - description: > - The number of failed attempts to connect to the MySQL server. - - - name: binlog - type: group - description: > - fields: - - name: cache.disk_use - type: long - description: > - - - name: cache.use - type: long - description: > - - - name: bytes - type: group - description: > - Bytes stats. - fields: - - name: received - format: bytes - type: long - description: > - The number of bytes received from all clients. - - - name: sent - type: long - format: bytes - description: > - The number of bytes sent to all clients. - - - name: threads - type: group - description: > - Threads stats. - fields: - - name: cached - type: long - description: > - The number of cached threads. - - - name: created - type: long - description: > - The number of created threads. - - - name: connected - type: long - description: > - The number of connected threads. - - - name: running - type: long - description: > - The number of running threads. - - - name: connections - type: long - description: > - - - name: created - type: group - description: > - fields: - - name: tmp.disk_tables - type: long - description: > - - - name: tmp.files - type: long - description: > - - - name: tmp.tables - type: long - description: > - - - name: delayed - type: group - description: > - fields: - - name: errors - type: long - description: > - - - name: insert_threads - type: long - description: > - - - name: writes - type: long - description: > - - - name: flush_commands - type: long - description: > - - - name: max_used_connections - type: long - description: > - - - name: open - type: group - description: > - fields: - - name: files - type: long - description: > - - - name: streams - type: long - description: > - - - name: tables - type: long - description: > - - - name: opened_tables - type: long - description: > - - - name: command - type: group - description: > - fields: - - name: delete - type: long - description: > - The number of DELETE queries since startup. - - - name: insert - type: long - description: > - The number of INSERT queries since startup. - - - name: select - type: long - description: > - The number of SELECT queries since startup. - - - name: update - type: long - description: > - The number of UPDATE queries since startup. - -- key: nginx - title: "Nginx" - description: > - Nginx server status metrics collected from various modules. - short_config: false - fields: - - name: nginx - type: group - description: > - `nginx` contains the metrics that were scraped from nginx. - fields: - - name: stubstatus - type: group - description: > - `stubstatus` contains the metrics that were scraped from the ngx_http_stub_status_module status page. - fields: - - name: hostname - type: keyword - description: > - Nginx hostname. - - name: active - type: long - description: > - The current number of active client connections including Waiting connections. - - name: accepts - type: long - description: > - The total number of accepted client connections. - - name: handled - type: long - description: > - The total number of handled client connections. - - name: dropped - type: long - description: > - The total number of dropped client connections. - - name: requests - type: long - description: > - The total number of client requests. - - name: current - type: long - description: > - The current number of client requests. - - name: reading - type: long - description: > - The current number of connections where Nginx is reading the request header. - - name: writing - type: long - description: > - The current number of connections where Nginx is writing the response back to the client. - - name: waiting - type: long - description: > - The current number of idle client connections waiting for a request. - -- key: php_fpm - title: "PHP_FPM" - description: > - beta[] - - PHP-FPM server status metrics collected from PHP-FPM. - short_config: false - fields: - - name: php_fpm - type: group - description: > - `php_fpm` contains the metrics that were obtained from PHP-FPM status - page call. - fields: - - name: pool - type: group - description: > - `pool` contains the metrics that were obtained from the PHP-FPM process - pool. - fields: - - name: name - type: keyword - description: > - The name of the pool. - - name: connections - type: group - description: > - Connection state specific statistics. - fields: - - name: accepted - type: long - description: > - The number of incoming requests that the PHP-FPM server has accepted; - when a connection is accepted it is removed from the listen queue. - - name: queued - type: long - description: > - The current number of connections that have been initiated, but not - yet accepted. If this value is non-zero it typically means that all - the available server processes are currently busy, and there are no - processes available to serve the next request. Raising - `pm.max_children` (provided the server can handle it) should help - keep this number low. This property follows from the fact that - PHP-FPM listens via a socket (TCP or file based), and thus inherits - some of the characteristics of sockets. - - name: processes - type: group - description: > - Process state specific statistics. - fields: - - name: idle - type: long - description: > - The number of servers in the `waiting to process` state (i.e. not - currently serving a page). This value should fall between the - `pm.min_spare_servers` and `pm.max_spare_servers` values when the - process manager is `dynamic`. - - name: active - type: long - description: > - The number of servers current processing a page - the minimum is `1` - (so even on a fully idle server, the result will be not read `0`). - - name: slow_requests - type: long - description: > - The number of times a request execution time has exceeded - `request_slowlog_timeout`. - -- key: postgresql - title: "PostgreSQL" - description: > - Metrics collected from PostgreSQL servers. - short_config: false - fields: - - name: postgresql - type: group - description: > - PostgreSQL metrics. - fields: - - name: activity - type: group - description: > - One document per server process, showing information related to the current - activity of that process, such as state and current query. Collected by - querying pg_stat_activity. - fields: - - name: database.oid - type: long - description: > - OID of the database this backend is connected to. - - name: database.name - type: keyword - description: > - Name of the database this backend is connected to. - - name: pid - type: long - description: > - Process ID of this backend. - - name: user.id - type: long - description: > - OID of the user logged into this backend. - - name: user.name - description: > - Name of the user logged into this backend. - - name: application_name - description: > - Name of the application that is connected to this backend. - - name: client.address - description: > - IP address of the client connected to this backend. - - name: client.hostname - description: > - Host name of the connected client, as reported by a reverse DNS lookup of client_addr. - - name: client.port - type: long - description: > - TCP port number that the client is using for communication with this - backend, or -1 if a Unix socket is used. - - name: backend_start - type: date - description: > - Time when this process was started, i.e., when the client connected to - the server. - - name: transaction_start - type: date - description: > - Time when this process' current transaction was started. - - name: query_start - type: date - description: > - Time when the currently active query was started, or if state is not - active, when the last query was started. - - name: state_change - type: date - description: > - Time when the state was last changed. - - name: waiting - type: boolean - description: > - True if this backend is currently waiting on a lock. - - name: state - description: > - Current overall state of this backend. Possible values are: - - * active: The backend is executing a query. - * idle: The backend is waiting for a new client command. - * idle in transaction: The backend is in a transaction, but is not - currently executing a query. - * idle in transaction (aborted): This state is similar to idle in - transaction, except one of the statements in the transaction caused - an error. - * fastpath function call: The backend is executing a fast-path function. - * disabled: This state is reported if track_activities is disabled in this backend. - - name: query - description: > - Text of this backend's most recent query. If state is active this field - shows the currently executing query. In all other states, it shows the - last query that was executed. - - - - name: bgwriter - type: group - description: > - Statistics about the background writer process's activity. Collected using the - pg_stat_bgwriter query. - fields: - - name: checkpoints.scheduled - type: long - description: > - Number of scheduled checkpoints that have been performed. - - name: checkpoints.requested - type: long - description: > - Number of requested checkpoints that have been performed. - - name: checkpoints.times.write.ms - type: float - description: > - Total amount of time that has been spent in the portion of checkpoint - processing where files are written to disk, in milliseconds. - - name: checkpoints.times.sync.ms - type: float - description: > - Total amount of time that has been spent in the portion of checkpoint - processing where files are synchronized to disk, in milliseconds. - - name: buffers.checkpoints - type: long - description: > - Number of buffers written during checkpoints. - - name: buffers.clean - type: long - description: > - Number of buffers written by the background writer. - - name: buffers.clean_full - type: long - description: > - Number of times the background writer stopped a cleaning scan because it - had written too many buffers. - - name: buffers.backend - type: long - description: > - Number of buffers written directly by a backend. - - name: buffers.backend_fsync - type: long - description: > - Number of times a backend had to execute its own fsync call (normally - the background writer handles those even when the backend does its own - write) - - name: buffers.allocated - type: long - description: > - Number of buffers allocated. - - name: stats_reset - type: date - description: > - Time at which these statistics were last reset. - - - name: database - type: group - description: > - One row per database, showing database-wide statistics. Collected by querying - pg_stat_database - fields: - - name: oid - type: long - description: > - OID of the database this backend is connected to. - - name: name - type: keyword - description: > - Name of the database this backend is connected to. - - name: number_of_backends - type: long - description: > - Number of backends currently connected to this database. - - name: transactions.commit - type: long - description: > - Number of transactions in this database that have been committed. - - name: transactions.rollback - type: long - description: > - Number of transactions in this database that have been rolled back. - - name: blocks.read - type: long - description: > - Number of disk blocks read in this database. - - name: blocks.hit - type: long - description: > - Number of times disk blocks were found already in the buffer cache, so - that a read was not necessary (this only includes hits in the PostgreSQL - buffer cache, not the operating system's file system cache). - - name: blocks.time.read.ms - type: long - description: > - Time spent reading data file blocks by backends in this database, in - milliseconds. - - name: blocks.time.write.ms - type: long - description: > - Time spent writing data file blocks by backends in this database, in - milliseconds. - - name: rows.returned - type: long - description: > - Number of rows returned by queries in this database. - - name: rows.fetched - type: long - description: > - Number of rows fetched by queries in this database. - - name: rows.inserted - type: long - description: > - Number of rows inserted by queries in this database. - - name: rows.updated - type: long - description: > - Number of rows updated by queries in this database. - - name: rows.deleted - type: long - description: > - Number of rows deleted by queries in this database. - - name: conflicts - type: long - description: > - Number of queries canceled due to conflicts with recovery in this - database. - - name: temporary.files - type: long - description: > - Number of temporary files created by queries in this database. All - temporary files are counted, regardless of why the temporary file was - created (e.g., sorting or hashing), and regardless of the log_temp_files - setting. - - name: temporary.bytes - type: long - description: > - Total amount of data written to temporary files by queries in this - database. All temporary files are counted, regardless of why the - temporary file was created, and regardless of the log_temp_files - setting. - - name: deadlocks - type: long - description: > - Number of deadlocks detected in this database. - - name: stats_reset - type: date - description: > - Time at which these statistics were last reset. - - -- key: prometheus - title: "Prometheus" - description: > - beta[] - - Stats collected from Prometheus. - short_config: false - fields: - - name: prometheus - type: group - description: > - fields: - - - name: stats - type: group - description: > - Stats about the Prometheus server. - fields: - - name: notifications - type: group - description: > - Notification stats. - fields: - - name: queue_length - type: long - description: > - Current queue length. - - name: dropped - type: long - description: > - Number of dropped queue events. - - name: processes.open_fds - type: long - description: > - Number of open file descriptors. - - name: storage.chunks_to_persist - type: long - description: > - Number of memory chunks that are not yet persisted to disk. - -- key: rabbitmq - title: "RabbitMQ" - description: > - experimental[] - - RabbitMQ module - fields: - - name: rabbitmq - type: group - description: > - fields: - - name: node - type: group - description: > - node - fields: - - name: disk.free.bytes - type: long - description: > - Disk free space in bytes. - format: bytes - - name: disk.free.limit.bytes - type: long - description: > - Point at which the disk alarm will go off. - format: bytes - - name: fd.total - type: long - description: > - File descriptors available. - - name: fd.used - type: long - description: > - Used file descriptors. - - name: gc.num.count - type: long - description: > - Number of GC operations. - - name: gc.reclaimed.bytes - type: long - description: > - GC bytes reclaimed. - format: bytes - - name: io.file_handle.open_attempt.avg.ms - type: long - description: > - File handle open avg time - - name: io.file_handle.open_attempt.count - type: long - description: > - File handle open attempts - - name: io.read.avg.ms - type: long - description: > - File handle read avg time - - name: io.read.bytes - type: long - description: > - Data read in bytes - format: bytes - - name: io.read.count - type: long - description: > - Data read operations - - name: io.reopen.count - type: long - description: > - Data reopen operations - - name: io.seek.avg.ms - type: long - description: > - Data seek avg time - - name: io.seek.count - type: long - description: > - Data seek operations - - name: io.sync.avg.ms - type: long - description: > - Data sync avg time - - name: io.sync.count - type: long - description: > - Data sync operations - - name: io.write.avg.ms - type: long - description: > - Data write avg time - - name: io.write.bytes - type: long - description: > - Data write in bytes - format: bytes - - name: io.write.count - type: long - description: > - Data write operations - - name: mem.limit.bytes - type: long - description: > - Point at which the memory alarm will go off. - format: bytes - - name: mem.used.bytes - type: long - description: > - Memory used in bytes. - - name: mnesia.disk.tx.count - type: long - description: > - Number of Mnesia transactions which have been performed that required writes to disk. - - name: mnesia.ram.tx.count - type: long - description: > - Number of Mnesia transactions which have been performed that did not require writes to disk. - - name: msg.store_read.count - type: long - description: > - Number of messages which have been read from the message store. - - name: msg.store_write.count - type: long - description: > - Number of messages which have been written to the message store. - - name: name - type: keyword - description: > - Node name - - name: proc.total - type: long - description: > - Maximum number of Erlang processes. - - name: proc.used - type: long - description: > - Number of Erlang processes in use. - - name: processors - type: long - description: > - Number of cores detected and usable by Erlang. - - name: queue.index.journal_write.count - type: long - description: > - Number of records written to the queue index journal. - - name: queue.index.read.count - type: long - description: > - Number of records read from the queue index. - - name: queue.index.write.count - type: long - description: > - Number of records written to the queue index. - - name: run.queue - type: long - description: > - Average number of Erlang processes waiting to run. - - name: socket.total - type: long - description: > - File descriptors available for use as sockets. - - name: socket.used - type: long - description: > - File descriptors used as sockets. - - name: type - type: keyword - description: > - Node type. - - name: uptime - type: long - description: > - Node uptime. - - - name: queue - type: group - description: > - queue - fields: - - name: name - type: keyword - description: > - The name of the queue with non-ASCII characters escaped as in C. - - name: vhost - type: keyword - description: > - Virtual host name with non-ASCII characters escaped as in C. - - name: durable - type: boolean - description: > - Whether or not the queue survives server restarts. - - name: auto_delete - type: boolean - description: > - Whether the queue will be deleted automatically when no longer used. - - name: exclusive - type: boolean - description: > - Whether the queue is exclusive (i.e. has owner_pid). - - name: node - type: keyword - description: > - Node name. - - name: state - type: keyword - description: > - The state of the queue. Normally 'running', but may be "{syncing, MsgCount}" if the queue is synchronising. Queues which are located on cluster nodes that are currently down will be shown with a status of 'down'. - - name: arguments.max_priority - type: long - description: > - Maximum number of priority levels for the queue to support. - - name: consumers.count - type: long - description: > - Number of consumers. - - name: consumers.utilisation.pct - type: long - format: percentage - description: > - Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count. - - name: messages.total.count - type: long - description: > - Sum of ready and unacknowledged messages (queue depth). - - name: messages.ready.count - type: long - description: > - Number of messages ready to be delivered to clients. - - name: messages.unacknowledged.count - type: long - description: > - Number of messages delivered to clients but not yet acknowledged. - - name: messages.persistent.count - type: long - description: > - Total number of persistent messages in the queue (will always be 0 for transient queues). - - name: memory.bytes - type: long - format: bytes - description: > - Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures. - - name: disk.reads.count - type: long - description: > - Total number of times messages have been read from disk by this queue since it started. - - name: disk.writes.count - type: long - description: > - Total number of times messages have been written to disk by this queue since it started. - -- key: redis - title: "Redis" - description: > - Redis metrics collected from Redis. - fields: - - name: redis - type: group - description: > - `redis` contains the information and statistics from Redis. - fields: - - name: info - type: group - description: > - `info` contains the information and statistics returned by the `INFO` command. - fields: - - name: clients - type: group - description: > - Redis client stats. - fields: - - name: connected - type: long - description: > - Number of client connections (excluding connections from slaves). - - - name: longest_output_list - type: long - description: > - Longest output list among current client connections. - - - name: biggest_input_buf - type: long - description: > - Biggest input buffer among current client connections. - - - name: blocked - type: long - description: > - Number of clients pending on a blocking call (BLPOP, BRPOP, BRPOPLPUSH). - - - name: cluster - type: group - description: > - Redis cluster information. - fields: - - name: enabled - type: boolean - description: > - Indicates that the Redis cluster is enabled. - - - name: cpu - type: group - description: > - Redis CPU stats - fields: - - name: used.sys - type: scaled_float - description: > - System CPU consumed by the Redis server. - - - name: used.sys_children - type: scaled_float - description: > - User CPU consumed by the Redis server. - - - name: used.user - type: scaled_float - description: > - System CPU consumed by the background processes. - - - name: used.user_children - type: scaled_float - description: > - User CPU consumed by the background processes. - - - name: memory - type: group - description: > - Redis memory stats. - fields: - - name: used.value - type: long - description: > - format: bytes - Used memory. - - - name: used.rss - type: long - format: bytes - description: > - Used memory rss. - - - name: used.peak - type: long - format: bytes - description: > - Used memory peak. - - - name: used.lua - type: long - format: bytes - description: > - Used memory lua. - - - name: allocator - type: keyword - description: > - Memory allocator. - - - name: persistence - type: group - description: > - Redis CPU stats. - fields: - - name: loading - type: boolean - description: - - - name: rdb - type: group - description: - fields: - - name: last_save.changes_since - type: long - description: - - - name: bgsave.in_progress - type: boolean - description: - - - name: last_save.time - type: long - description: - - - name: bgsave.last_status - type: keyword - description: - - - name: bgsave.last_time.sec - type: long - description: - - - name: bgsave.current_time.sec - type: long - description: - - - name: aof - type: group - description: - fields: - - name: enabled - type: boolean - description: - - - name: rewrite.in_progress - type: boolean - description: - - - name: rewrite.scheduled - type: boolean - description: - - - name: rewrite.last_time.sec - type: long - description: - - - name: rewrite.current_time.sec - type: long - description: - - - name: bgrewrite.last_status - type: keyword - description: - - - name: write.last_status - type: keyword - description: - - - name: replication - type: group - description: > - Replication - fields: - - name: role - type: keyword - description: - - - name: connected_slaves - type: long - description: - - - name: master_offset - type: long - description: - - - name: backlog.active - type: long - description: - - - name: backlog.size - type: long - description: - - - name: backlog.first_byte_offset - type: long - description: - - - name: backlog.histlen - type: long - description: - - - name: server - type: group - description: > - Server info - fields: - - name: version - type: keyword - description: - - - name: git_sha1 - type: keyword - description: - - - name: git_dirty - type: keyword - description: - - - name: build_id - type: keyword - description: - - - name: mode - type: keyword - description: - - - name: os - type: keyword - description: - - - name: arch_bits - type: keyword - description: - - - name: multiplexing_api - type: keyword - description: - - - name: gcc_version - type: keyword - description: - - - name: process_id - type: long - description: - - - name: run_id - type: keyword - description: - - - name: tcp_port - type: long - description: - - - name: uptime - type: long - description: - - - name: hz - type: long - description: - - - name: lru_clock - type: long - description: - - - name: config_file - type: keyword - description: - - - name: stats - type: group - description: > - Redis stats. - fields: - - name: connections.received - type: long - description: - Total number of connections received. - - - name: connections.rejected - type: long - description: - Total number of connections rejected. - - - name: commands_processed - type: long - description: - Total number of commands preocessed. - - - name: net.input.bytes - type: long - description: - Total network input in bytes. - - - name: net.output.bytes - type: long - description: - Total network output in bytes. - - - name: instantaneous.ops_per_sec - type: long - description: - - - name: instantaneous.input_kbps - type: scaled_float - description: - - - name: instantaneous.output_kbps - type: scaled_float - description: - - - name: sync.full - type: long - description: - - - name: sync.partial.ok - type: long - description: - - - name: sync.partial.err - type: long - description: - - - name: keys.expired - type: long - description: - - - name: keys.evicted - type: long - description: - - - name: keyspace.hits - type: long - description: - - - name: keyspace.misses - type: long - description: - - - name: pubsub.channels - type: long - description: - - - name: pubsub.patterns - type: long - description: - - - name: latest_fork_usec - type: long - description: - - - name: migrate_cached_sockets - type: long - description: - - - - name: keyspace - type: group - description: > - `keyspace` contains the information about the keyspaces returned by the `INFO` command. - fields: - - name: id - type: keyword - description: > - Keyspace identifier. - - - name: avg_ttl - type: long - description: > - Average ttl. - - - name: keys - type: long - description: > - Number of keys in the keyspace. - - - name: expires - type: long - description: > - -- key: system - title: "System" - description: > - System status metrics, like CPU and memory usage, that are collected from the operating system. - short_config: true - fields: - - name: system - type: group - description: > - `system` contains local system metrics. - fields: - - name: core - type: group - description: > - `system-core` contains CPU metrics for a single core of a multi-core system. - fields: - - name: id - type: long - description: > - CPU Core number. - - # Percentages - - name: user.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in user space. - - - name: user.ticks - type: long - description: > - The amount of CPU time spent in user space. - - - name: system.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in kernel space. - - - name: system.ticks - type: long - description: > - The amount of CPU time spent in kernel space. - - - name: nice.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent on low-priority processes. - - - name: nice.ticks - type: long - description: > - The amount of CPU time spent on low-priority processes. - - - name: idle.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent idle. - - - name: idle.ticks - type: long - description: > - The amount of CPU time spent idle. - - - name: iowait.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in wait (on disk). - - - name: iowait.ticks - type: long - description: > - The amount of CPU time spent in wait (on disk). - - - name: irq.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent servicing and handling hardware interrupts. - - - name: irq.ticks - type: long - description: > - The amount of CPU time spent servicing and handling hardware interrupts. - - - name: softirq.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent servicing and handling software interrupts. - - - name: softirq.ticks - type: long - description: > - The amount of CPU time spent servicing and handling software interrupts. - - - name: steal.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor - was servicing another processor. - Available only on Unix. - - - name: steal.ticks - type: long - description: > - The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor - was servicing another processor. - Available only on Unix. - - - name: cpu - type: group - description: > - `cpu` contains local CPU stats. - fields: - - name: cores - type: long - description: > - The number of CPU cores present on the host. The non-normalized - percentages will have a maximum value of `100% * cores`. The - normalized percentages already take this value into account and have - a maximum value of 100%. - - # Percentages - - name: user.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in user space. On multi-core systems, - you can have percentages that are greater than 100%. For example, if 3 - cores are at 60% use, then the `system.cpu.user.pct` will be 180%. - - - name: system.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in kernel space. - - - name: nice.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent on low-priority processes. - - - name: idle.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent idle. - - - name: iowait.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in wait (on disk). - - - name: irq.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent servicing and handling hardware interrupts. - - - name: softirq.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent servicing and handling software interrupts. - - - name: steal.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor - was servicing another processor. - Available only on Unix. - - - name: total.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in non-idle state. - - # Normalized Percentages - - name: user.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in user space. - - - name: system.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in kernel space. - - - name: nice.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent on low-priority processes. - - - name: idle.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent idle. - - - name: iowait.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in wait (on disk). - - - name: irq.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent servicing and handling hardware interrupts. - - - name: softirq.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent servicing and handling software interrupts. - - - name: steal.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor - was servicing another processor. - Available only on Unix. - - - name: total.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent in non-idle state. - - - # Ticks - - name: user.ticks - type: long - description: > - The amount of CPU time spent in user space. - - - name: system.ticks - type: long - description: > - The amount of CPU time spent in kernel space. - - - name: nice.ticks - type: long - description: > - The amount of CPU time spent on low-priority processes. - - - name: idle.ticks - type: long - description: > - The amount of CPU time spent idle. - - - name: iowait.ticks - type: long - description: > - The amount of CPU time spent in wait (on disk). - - - name: irq.ticks - type: long - description: > - The amount of CPU time spent servicing and handling hardware interrupts. - - - name: softirq.ticks - type: long - description: > - The amount of CPU time spent servicing and handling software interrupts. - - - name: steal.ticks - type: long - description: > - The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor - was servicing another processor. - Available only on Unix. - - - name: diskio - type: group - description: > - `disk` contains disk IO metrics collected from the operating system. - fields: - - name: name - type: keyword - example: sda1 - description: > - The disk name. - - - name: serial_number - type: keyword - description: > - The disk's serial number. This may not be provided by all operating - systems. - - - name: read.count - type: long - description: > - The total number of reads completed successfully. - - - name: write.count - type: long - description: > - The total number of writes completed successfully. - - - name: read.bytes - type: long - format: bytes - description: > - The total number of bytes read successfully. On Linux this is - the number of sectors read multiplied by an assumed sector size of 512. - - - name: write.bytes - type: long - format: bytes - description: > - The total number of bytes written successfully. On Linux this is - the number of sectors written multiplied by an assumed sector size of - 512. - - - name: read.time - type: long - description: > - The total number of milliseconds spent by all reads. - - - name: write.time - type: long - description: > - The total number of milliseconds spent by all writes. - - - name: io.time - type: long - description: > - The total number of of milliseconds spent doing I/Os. - - - name: iostat.read.request.merges_per_sec - type: float - description: > - The number of read requests merged per second that were queued to the device. - - - name: iostat.write.request.merges_per_sec - type: float - description: > - The number of write requests merged per second that were queued to the device. - - - name: iostat.read.request.per_sec - type: float - description: > - The number of read requests that were issued to the device per second - - - name: iostat.write.request.per_sec - type: float - description: > - The number of write requests that were issued to the device per second - - - name: iostat.read.per_sec.bytes - type: float - description: > - The number of Bytes read from the device per second. - format: bytes - - - name: iostat.write.per_sec.bytes - type: float - description: > - The number of Bytes write from the device per second. - format: bytes - - - name: iostat.request.avg_size - type: float - description: > - The average size (in sectors) of the requests that were issued to the device. - - - name: iostat.queue.avg_size - type: float - description: > - The average queue length of the requests that were issued to the device. - - - name: iostat.await - type: float - description: > - The average queue length of the requests that were issued to the device. - - - name: iostat.service_time - type: float - description: > - The average service time (in milliseconds) for I/O requests that were issued to the device. - - - name: iostat.busy - type: float - description: > - Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. - - - name: filesystem - type: group - description: > - `filesystem` contains local filesystem stats. - fields: - - name: available - type: long - format: bytes - description: > - The disk space available to an unprivileged user in bytes. - - name: device_name - type: keyword - description: > - The disk name. For example: `/dev/disk1` - - name: type - type: keyword - description: > - The disk type. For example: `ext4` - - name: mount_point - type: keyword - description: > - The mounting point. For example: `/` - - name: files - type: long - description: > - The total number of file nodes in the file system. - - name: free - type: long - format: bytes - description: > - The disk space available in bytes. - - name: free_files - type: long - description: > - The number of free file nodes in the file system. - - name: total - type: long - format: bytes - description: > - The total disk space in bytes. - - name: used.bytes - type: long - format: bytes - description: > - The used disk space in bytes. - - name: used.pct - type: scaled_float - format: percent - description: > - The percentage of used disk space. - - - - - name: fsstat - type: group - description: > - `system.fsstat` contains filesystem metrics aggregated from all mounted - filesystems, similar with what `df -a` prints out. - fields: - - name: count - type: long - description: Number of file systems found. - - name: total_files - type: long - description: Total number of files. - - name: total_size - format: bytes - type: group - description: Nested file system docs. - fields: - - name: free - type: long - format: bytes - description: > - Total free space. - - name: used - type: long - format: bytes - description: > - Total used space. - - name: total - type: long - format: bytes - description: > - Total space (used plus free). - - - name: load - type: group - description: > - CPU load averages. - fields: - - name: "1" - type: scaled_float - scaling_factor: 100 - description: > - Load average for the last minute. - - name: "5" - type: scaled_float - scaling_factor: 100 - description: > - Load average for the last 5 minutes. - - name: "15" - type: scaled_float - scaling_factor: 100 - description: > - Load average for the last 15 minutes. - - - name: "norm.1" - type: scaled_float - scaling_factor: 100 - description: > - Load for the last minute divided by the number of cores. - - - name: "norm.5" - type: scaled_float - scaling_factor: 100 - description: > - Load for the last 5 minutes divided by the number of cores. - - - name: "norm.15" - type: scaled_float - scaling_factor: 100 - description: > - Load for the last 15 minutes divided by the number of cores. - - - name: "cores" - type: long - description: > - The number of CPU cores present on the host. - - - name: memory - type: group - description: > - `memory` contains local memory stats. - fields: - - name: total - type: long - format: bytes - description: > - Total memory. - - - name: used.bytes - type: long - format: bytes - description: > - Used memory. - - - name: free - type: long - format: bytes - description: > - The total amount of free memory in bytes. This value does not include memory consumed by system caches and - buffers (see system.memory.actual.free). - - - name: used.pct - type: scaled_float - format: percent - description: > - The percentage of used memory. - - - name: actual - type: group - description: > - Actual memory used and free. - fields: - - - name: used.bytes - type: long - format: bytes - description: > - Actual used memory in bytes. It represents the difference between the total and the available memory. The - available memory depends on the OS. For more details, please check `system.actual.free`. - - - name: free - type: long - format: bytes - description: > - Actual free memory in bytes. It is calculated based on the OS. On Linux it consists of the free memory - plus caches and buffers. On OSX it is a sum of free memory and the inactive memory. On Windows, it is equal - to `system.memory.free`. - - - name: used.pct - type: scaled_float - format: percent - description: > - The percentage of actual used memory. - - - name: swap - type: group - prefix: "[float]" - description: This group contains statistics related to the swap memory usage on the system. - fields: - - name: total - type: long - format: bytes - description: > - Total swap memory. - - - name: used.bytes - type: long - format: bytes - description: > - Used swap memory. - - - name: free - type: long - format: bytes - description: > - Available swap memory. - - - name: used.pct - type: scaled_float - format: percent - description: > - The percentage of used swap memory. - - - name: network - type: group - description: > - `network` contains network IO metrics for a single network interface. - fields: - - name: name - type: keyword - example: eth0 - description: > - The network interface name. - - - name: out.bytes - type: long - format: bytes - description: > - The number of bytes sent. - - - name: in.bytes - type: long - format: bytes - description: > - The number of bytes received. - - - name: out.packets - type: long - description: > - The number of packets sent. - - - name: in.packets - type: long - description: > - The number or packets received. - - - name: in.errors - type: long - description: > - The number of errors while receiving. - - - name: out.errors - type: long - description: > - The number of errors while sending. - - - name: in.dropped - type: long - description: > - The number of incoming packets that were dropped. - - - name: out.dropped - type: long - description: > - The number of outgoing packets that were dropped. This value is always - 0 on Darwin and BSD because it is not reported by the operating system. - - - name: process - type: group - description: > - `process` contains process metadata, CPU metrics, and memory metrics. - fields: - - name: name - type: keyword - description: > - The process name. - - name: state - type: keyword - description: > - The process state. For example: "running". - - name: pid - type: long - description: > - The process pid. - - name: ppid - type: long - description: > - The process parent pid. - - name: pgid - type: long - description: > - The process group id. - - name: cmdline - type: keyword - description: > - The full command-line used to start the process, including the - arguments separated by space. - - name: username - type: keyword - description: > - The username of the user that created the process. If the username - cannot be determined, the field will contain the user's - numeric identifier (UID). On Windows, this field includes the user's - domain and is formatted as `domain\username`. - - name: cwd - type: keyword - description: > - The current working directory of the process. This field is only - available on Linux. - - name: env - type: object - object_type: keyword - description: > - The environment variables used to start the process. The data is - available on FreeBSD, Linux, and OS X. - - name: cpu - type: group - prefix: "[float]" - description: CPU-specific statistics per process. - fields: - - name: user - type: long - description: > - The amount of CPU time the process spent in user space. - - name: total.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent by the process since the last update. Its value is similar to the - %CPU value of the process displayed by the top command on Unix systems. - - name: total.norm.pct - type: scaled_float - format: percent - description: > - The percentage of CPU time spent by the process since the last event. - This value is normalized by the number of CPU cores and it ranges - from 0 to 100%. - - name: system - type: long - description: > - The amount of CPU time the process spent in kernel space. - - name: total.ticks - type: long - description: > - The total CPU time spent by the process. - - name: start_time - type: date - description: > - The time when the process was started. - - name: memory - type: group - description: Memory-specific statistics per process. - prefix: "[float]" - fields: - - name: size - type: long - format: bytes - description: > - The total virtual memory the process has. - - name: rss.bytes - type: long - format: bytes - description: > - The Resident Set Size. The amount of memory the process occupied in main memory (RAM). - - name: rss.pct - type: scaled_float - format: percent - description: > - The percentage of memory the process occupied in main memory (RAM). - - name: share - type: long - format: bytes - description: > - The shared memory the process uses. - - name: fd - type: group - description: > - File descriptor usage metrics. This set of metrics is available for - Linux and FreeBSD. - prefix: "[float]" - fields: - - name: open - type: long - description: The number of file descriptors open by the process. - - name: limit.soft - type: long - description: > - The soft limit on the number of file descriptors opened by the - process. The soft limit can be changed by the process at any time. - - name: limit.hard - type: long - description: > - The hard limit on the number of file descriptors opened by the - process. The hard limit can only be raised by root. - - name: cgroup - type: group - description: > - Metrics and limits from the cgroup of which the task is a member. - cgroup metrics are reported when the process has membership in a - non-root cgroup. These metrics are only available on Linux. - fields: - - name: id - type: keyword - description: > - The ID common to all cgroups associated with this task. - If there isn't a common ID used by all cgroups this field will be - absent. - - - name: path - type: keyword - description: > - The path to the cgroup relative to the cgroup subsystem's mountpoint. - If there isn't a common path used by all cgroups this field will be - absent. - - - name: cpu - type: group - description: > - The cpu subsystem schedules CPU access for tasks in the cgroup. - Access can be controlled by two separate schedulers, CFS and RT. - CFS stands for completely fair scheduler which proportionally - divides the CPU time between cgroups based on weight. RT stands for - real time scheduler which sets a maximum amount of CPU time that - processes in the cgroup can consume during a given period. - - fields: - - name: id - type: keyword - description: ID of the cgroup. - - - name: path - type: keyword - description: > - Path to the cgroup relative to the cgroup subsystem's - mountpoint. - - - name: cfs.period.us - type: long - description: > - Period of time in microseconds for how regularly a - cgroup's access to CPU resources should be reallocated. - - - name: cfs.quota.us - type: long - description: > - Total amount of time in microseconds for which all - tasks in a cgroup can run during one period (as defined by - cfs.period.us). - - - name: cfs.shares - type: long - description: > - An integer value that specifies a relative share of CPU time - available to the tasks in a cgroup. The value specified in the - cpu.shares file must be 2 or higher. - - - name: rt.period.us - type: long - description: > - Period of time in microseconds for how regularly a cgroup's - access to CPU resources is reallocated. - - - name: rt.runtime.us - type: long - description: > - Period of time in microseconds for the longest continuous period - in which the tasks in a cgroup have access to CPU resources. - - - name: stats.periods - type: long - description: > - Number of period intervals (as specified in cpu.cfs.period.us) - that have elapsed. - - - name: stats.throttled.periods - type: long - description: > - Number of times tasks in a cgroup have been throttled (that is, - not allowed to run because they have exhausted all of the - available time as specified by their quota). - - - name: stats.throttled.ns - type: long - description: > - The total time duration (in nanoseconds) for which tasks in a - cgroup have been throttled. - - - name: cpuacct - type: group - description: CPU accounting metrics. - fields: - - name: id - type: keyword - description: ID of the cgroup. - - - name: path - type: keyword - description: > - Path to the cgroup relative to the cgroup subsystem's - mountpoint. - - - name: total.ns - type: long - description: > - Total CPU time in nanoseconds consumed by all tasks in the - cgroup. - - - name: stats.user.ns - type: long - description: CPU time consumed by tasks in user mode. - - - name: stats.system.ns - type: long - description: CPU time consumed by tasks in user (kernel) mode. - - - name: percpu - type: object - object_type: long - description: > - CPU time (in nanoseconds) consumed on each CPU by all tasks in - this cgroup. - - - name: memory - type: group - description: Memory limits and metrics. - fields: - - name: id - type: keyword - description: ID of the cgroup. - - - name: path - type: keyword - description: > - Path to the cgroup relative to the cgroup subsystem's mountpoint. - - - name: mem.usage.bytes - type: long - format: bytes - description: > - Total memory usage by processes in the cgroup (in bytes). - - - name: mem.usage.max.bytes - type: long - format: bytes - description: > - The maximum memory used by processes in the cgroup (in bytes). - - - name: mem.limit.bytes - type: long - format: bytes - description: > - The maximum amount of user memory in bytes (including file - cache) that tasks in the cgroup are allowed to use. - - - name: mem.failures - type: long - description: > - The number of times that the memory limit (mem.limit.bytes) was - reached. - - - name: memsw.usage.bytes - type: long - format: bytes - description: > - The sum of current memory usage plus swap space used by - processes in the cgroup (in bytes). - - - name: memsw.usage.max.bytes - type: long - format: bytes - description: > - The maximum amount of memory and swap space used by processes in - the cgroup (in bytes). - - - name: memsw.limit.bytes - type: long - format: bytes - description: > - The maximum amount for the sum of memory and swap usage - that tasks in the cgroup are allowed to use. - - - name: memsw.failures - type: long - description: > - The number of times that the memory plus swap space limit - (memsw.limit.bytes) was reached. - - - name: kmem.usage.bytes - type: long - format: bytes - description: > - Total kernel memory usage by processes in the cgroup (in bytes). - - - name: kmem.usage.max.bytes - type: long - format: bytes - description: > - The maximum kernel memory used by processes in the cgroup (in - bytes). - - - name: kmem.limit.bytes - type: long - format: bytes - description: > - The maximum amount of kernel memory that tasks in the cgroup are - allowed to use. - - - name: kmem.failures - type: long - description: > - The number of times that the memory limit (kmem.limit.bytes) was - reached. - - - name: kmem_tcp.usage.bytes - type: long - format: bytes - description: > - Total memory usage for TCP buffers in bytes. - - - name: kmem_tcp.usage.max.bytes - type: long - format: bytes - description: > - The maximum memory used for TCP buffers by processes in the - cgroup (in bytes). - - - name: kmem_tcp.limit.bytes - type: long - format: bytes - description: > - The maximum amount of memory for TCP buffers that tasks in the - cgroup are allowed to use. - - - name: kmem_tcp.failures - type: long - description: > - The number of times that the memory limit (kmem_tcp.limit.bytes) - was reached. - - - name: stats.active_anon.bytes - type: long - format: bytes - description: > - Anonymous and swap cache on active least-recently-used (LRU) - list, including tmpfs (shmem), in bytes. - - - name: stats.active_file.bytes - type: long - format: bytes - description: File-backed memory on active LRU list, in bytes. - - - name: stats.cache.bytes - type: long - format: bytes - description: Page cache, including tmpfs (shmem), in bytes. - - - name: stats.hierarchical_memory_limit.bytes - type: long - format: bytes - description: > - Memory limit for the hierarchy that contains the memory cgroup, - in bytes. - - - name: stats.hierarchical_memsw_limit.bytes - type: long - format: bytes - description: > - Memory plus swap limit for the hierarchy that contains the - memory cgroup, in bytes. - - - name: stats.inactive_anon.bytes - type: long - format: bytes - description: > - Anonymous and swap cache on inactive LRU list, including tmpfs - (shmem), in bytes - - - name: stats.inactive_file.bytes - type: long - format: bytes - description: > - File-backed memory on inactive LRU list, in bytes. - - - name: stats.mapped_file.bytes - type: long - format: bytes - description: > - Size of memory-mapped mapped files, including tmpfs (shmem), - in bytes. - - - name: stats.page_faults - type: long - description: > - Number of times that a process in the cgroup triggered a page - fault. - - - name: stats.major_page_faults - type: long - description: > - Number of times that a process in the cgroup triggered a major - fault. "Major" faults happen when the kernel actually has to - read the data from disk. - - - name: stats.pages_in - type: long - description: > - Number of pages paged into memory. This is a counter. - - - name: stats.pages_out - type: long - description: > - Number of pages paged out of memory. This is a counter. - - - name: stats.rss.bytes - type: long - format: bytes - description: > - Anonymous and swap cache (includes transparent hugepages), not - including tmpfs (shmem), in bytes. - - - name: stats.rss_huge.bytes - type: long - format: bytes - description: > - Number of bytes of anonymous transparent hugepages. - - - name: stats.swap.bytes - type: long - format: bytes - description: > - Swap usage, in bytes. - - - name: stats.unevictable.bytes - type: long - format: bytes - description: > - Memory that cannot be reclaimed, in bytes. - - - name: blkio - type: group - description: Block IO metrics. - fields: - - name: id - type: keyword - description: ID of the cgroup. - - - name: path - type: keyword - description: > - Path to the cgroup relative to the cgroup subsystems mountpoint. - - - name: total.bytes - type: long - format: bytes - description: > - Total number of bytes transferred to and from all block devices - by processes in the cgroup. - - - name: total.ios - type: long - description: > - Total number of I/O operations performed on all devices - by processes in the cgroup as seen by the throttling policy. - - - name: process.summary - title: Process Summary - type: group - description: > - Summary metrics for the processes running on the host. - fields: - - name: total - type: long - description: > - Total number of processes on this host. - - name: running - type: long - description: > - Number of running processes on this host. - - name: idle - type: long - description: > - Number of idle processes on this host. - - name: sleeping - type: long - description: > - Number of sleeping processes on this host. - - name: stopped - type: long - description: > - Number of stopped processes on this host. - - name: zombie - type: long - description: > - Number of zombie processes on this host. - - name: unknown - type: long - description: > - Number of processes for which the state couldn't be retrieved or is unknown. - - - name: socket - type: group - description: > - TCP sockets that are active. - fields: - - name: direction - type: keyword - example: incoming - description: > - How the socket was initiated. Possible values are incoming, outgoing, - or listening. - - - name: family - type: keyword - example: ipv4 - description: > - Address family. - - - name: local.ip - type: ip - example: 192.0.2.1 or 2001:0DB8:ABED:8536::1 - description: > - Local IP address. This can be an IPv4 or IPv6 address. - - - name: local.port - type: long - example: 22 - description: > - Local port. - - - name: remote.ip - type: ip - example: 192.0.2.1 or 2001:0DB8:ABED:8536::1 - description: > - Remote IP address. This can be an IPv4 or IPv6 address. - - - name: remote.port - type: long - example: 22 - description: > - Remote port. - - - name: remote.host - type: keyword - example: 76-211-117-36.nw.example.com. - description: > - PTR record associated with the remote IP. It is obtained via reverse - IP lookup. - - - name: remote.etld_plus_one - type: keyword - example: example.com. - description: > - The effective top-level domain (eTLD) of the remote host plus one more - label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". - The data for determining the eTLD comes from an embedded copy of the data - from http://publicsuffix.org. - - - name: remote.host_error - type: keyword - description: > - Error describing the cause of the reverse lookup failure. - - - name: process.pid - type: long - description: > - ID of the process that opened the socket. - - - name: process.command - type: keyword - description: > - Name of the command (limited to 20 chars by the OS). - - - name: process.cmdline - type: keyword - description: > - - - name: process.exe - type: keyword - description: > - Absolute path to the executable. - - - name: user.id - type: long - description: > - UID of the user running the process. - - - name: user.name - type: keyword - description: > - Name of the user running the process. - - - name: uptime - type: group - description: > - `uptime` contains the operating system uptime metric. - fields: - - name: duration.ms - type: long - format: duration - input_format: milliseconds - description: > - The OS uptime in milliseconds. - -- key: vsphere - title: "vSphere" - description: > - vSphere module - fields: - - name: vsphere - type: group - description: > - fields: - - name: datastore - type: group - description: > - datastore - fields: - - name: datacenter - type: keyword - description: > - Datacenter name - - name: name - type: keyword - description: > - Datastore name - - name: fstype - type: keyword - description: > - Filesystem type - - name: capacity.total.bytes - type: long - description: > - Total bytes of the datastore - format: bytes - - name: capacity.free.bytes - type: long - description: > - Free bytes of the datastore - format: bytes - - name: capacity.used.bytes - type: long - description: > - Used bytes of the datastore - format: bytes - - name: capacity.used.pct - type: long - description: > - Used percent of the datastore - format: percent - - - - name: host - type: group - description: > - host - fields: - - name: datacenter - type: keyword - description: > - Datacenter name - - name: name - type: keyword - description: > - Host name - - name: cpu.used.mhz - type: long - description: > - Used CPU in Mhz - - name: cpu.total.mhz - type: long - description: > - Total CPU in Mhz - - name: cpu.free.mhz - type: long - description: > - Free CPU in Mhz - - name: memory.used.bytes - type: long - description: > - Used Memory in bytes - format: bytes - - name: memory.total.bytes - type: long - description: > - Total Memory in bytes - format: bytes - - name: memory.free.bytes - type: long - description: > - Free Memory in bytes - format: bytes - - - name: virtualmachine - type: group - description: > - virtualmachine - fields: - - name: datacenter - type: keyword - description: > - Datacenter name - - name: name - type: keyword - description: > - Virtual Machine name - - name: cpu.used.mhz - type: long - description: > - Used CPU in Mhz - - name: memory.used.guest.bytes - type: long - description: > - Used Memory of Guest in bytes - format: bytes - - name: memory.used.host.bytes - type: long - description: > - Used Memory of Host in bytes - format: bytes - - name: memory.total.guest.bytes - type: long - description: > - Total Memory of Guest in bytes - format: bytes - - name: memory.free.guest.bytes - type: long - description: > - Free Memory of Guest in bytes - format: bytes - - name: custom_fields - type: object - object_type: keyword - descripton: > - Custom fields - -- key: windows - title: "Windows" - description: > - beta[] - Module for Windows - short_config: false - fields: - - name: windows - type: group - description: > - fields: - -- key: zookeeper - title: "ZooKeeper" - description: > - ZooKeeper metrics collected by the four-letter monitoring commands. - short_config: false - fields: - - name: zookeeper - type: group - description: > - `zookeeper` contains the metrics reported by ZooKeeper - commands. - fields: - - name: mntr - type: group - description: > - `mntr` contains the metrics reported by the four-letter `mntr` - command. - fields: - - name: hostname - type: keyword - description: > - ZooKeeper hostname. - - name: approximate_data_size - type: long - description: > - Approximate size of ZooKeeper data. - - name: latency.avg - type: long - description: > - Average latency between ensemble hosts in milliseconds. - - name: ephemerals_count - type: long - description: > - Number of ephemeral znodes. - - name: followers - type: long - description: > - Number of followers seen by the current host. - - name: max_file_descriptor_count - type: long - description: > - Maximum number of file descriptors allowed for the ZooKeeper process. - - name: latency.max - type: long - description: > - Maximum latency in milliseconds. - - name: latency.min - type: long - description: > - Minimum latency in milliseconds. - - name: num_alive_connections - type: long - description: > - Number of connections to ZooKeeper that are currently alive. - - name: open_file_descriptor_count - type: long - description: > - Number of file descriptors open by the ZooKeeper process. - - name: outstanding_requests - type: long - description: > - Number of outstanding requests that need to be processed by the cluster. - - name: packets.received - type: long - description: > - Number of ZooKeeper network packets received. - - name: packets.sent - type: long - description: > - Number of ZooKeeper network packets sent. - - name: pending_syncs - type: long - description: > - Number of pending syncs to carry out to ZooKeeper ensemble followers. - - name: server_state - type: keyword - description: > - Role in the ZooKeeper ensemble. - - name: synced_followers - type: long - description: > - Number of synced followers reported when a node server_state is leader. - - name: version - type: keyword - description: > - ZooKeeper version and build string reported. - - name: watch_count - type: long - description: > - Number of watches currently set on the local ZooKeeper process. - - name: znode_count - type: long - description: > - Number of znodes reported by the local ZooKeeper process. - - diff --git a/libbeat/kibana/transform.go b/libbeat/kibana/transform.go index 6b334171efc..d2007cc7ef0 100644 --- a/libbeat/kibana/transform.go +++ b/libbeat/kibana/transform.go @@ -74,6 +74,15 @@ func (t *Transformer) transformFields(commonFields common.Fields, path string) { f.Searchable = &truthy f.Aggregatable = &truthy t.add(f) + + if f.MultiFields != nil { + path := f.Path + for _, mf := range f.MultiFields { + f.Type = mf.Type + f.Path = path + "." + mf.Name + t.add(f) + } + } } } } diff --git a/libbeat/kibana/transform_test.go b/libbeat/kibana/transform_test.go index 6aff5707d32..380dc77470d 100644 --- a/libbeat/kibana/transform_test.go +++ b/libbeat/kibana/transform_test.go @@ -283,6 +283,7 @@ func TestTransformFieldFormatMap(t *testing.T) { assert.Equal(t, test.expected, out, fmt.Sprintf("Failed for idx %v", idx)) } } + func TestTransformGroupAndEnabled(t *testing.T) { tests := []struct { commonFields common.Fields @@ -351,3 +352,22 @@ func TestTransformGroupAndEnabled(t *testing.T) { } } } + +func TestTransformMultiField(t *testing.T) { + f := common.Field{ + Name: "context", + Type: "", + MultiFields: common.Fields{ + common.Field{Name: "keyword", Type: "keyword"}, + common.Field{Name: "text", Type: "text"}, + }, + } + trans := NewTransformer("name", "title", common.Fields{f}) + out := trans.TransformFields()["fields"].([]common.MapStr) + assert.Equal(t, "context", out[0]["name"]) + assert.Equal(t, "context.keyword", out[1]["name"]) + assert.Equal(t, "context.text", out[2]["name"]) + assert.Equal(t, "string", out[0]["type"]) + assert.Equal(t, "string", out[1]["type"]) + assert.Equal(t, "string", out[2]["type"]) +} From b3ca270e34aff72684742141fe32b2a71586bdb9 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Tue, 3 Oct 2017 11:25:26 +0200 Subject: [PATCH 67/96] Allow values to be set from outside (#5297) Remove hardcoded handling for values and allow setting them from outside. If not set, use default values. --- libbeat/kibana/transform.go | 31 +++++++++++++++++++++---------- libbeat/kibana/transform_test.go | 20 ++++++++------------ 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/libbeat/kibana/transform.go b/libbeat/kibana/transform.go index d2007cc7ef0..0972018bc78 100644 --- a/libbeat/kibana/transform.go +++ b/libbeat/kibana/transform.go @@ -63,16 +63,7 @@ func (t *Transformer) transformFields(commonFields common.Fields, path string) { t.transformFields(f.Fields, f.Path) } } else { - // set default values (as done in python script) t.keys[f.Path] = true - - truthy := true - falsy := false - f.Index = &truthy - f.Analyzed = &falsy - f.DocValues = &truthy - f.Searchable = &truthy - f.Aggregatable = &truthy t.add(f) if f.MultiFields != nil { @@ -99,7 +90,7 @@ func (t *Transformer) add(f common.Field) { func transformField(f common.Field) (common.MapStr, common.MapStr) { field := common.MapStr{ "name": f.Path, - "count": 0, + "count": f.Count, "scripted": false, "indexed": getVal(f.Index, true), "analyzed": getVal(f.Analyzed, false), @@ -134,6 +125,26 @@ func transformField(f common.Field) (common.MapStr, common.MapStr) { return field, format } +func setVal(f common.MapStr, attr string, p *bool, def bool) { + if p != nil { + f[attr] = *p + } + if f[attr] == nil { + f[attr] = def + } +} + +func setAggregatable(f common.MapStr, c common.Field) { + attr := "aggregatable" + if c.Aggregatable != nil { + f[attr] = *c.Aggregatable + } else if c.Type == "text" { + f[attr] = false + } else if f[attr] == nil { + f[attr] = true + } +} + func getVal(valP *bool, def bool) bool { if valP != nil { return *valP diff --git a/libbeat/kibana/transform_test.go b/libbeat/kibana/transform_test.go index 380dc77470d..65856c78186 100644 --- a/libbeat/kibana/transform_test.go +++ b/libbeat/kibana/transform_test.go @@ -161,34 +161,30 @@ func TestTransformMisc(t *testing.T) { attr string }{ {commonField: common.Field{}, expected: 0, attr: "count"}, + {commonField: common.Field{Count: 4}, expected: 4, attr: "count"}, - // searchable always set to true except for meta fields + // searchable {commonField: common.Field{}, expected: true, attr: "searchable"}, {commonField: common.Field{Searchable: &truthy}, expected: true, attr: "searchable"}, - {commonField: common.Field{Searchable: &falsy}, expected: true, attr: "searchable"}, + {commonField: common.Field{Searchable: &falsy}, expected: false, attr: "searchable"}, - // aggregatable always set to true except for meta fields or type text + // aggregatable {commonField: common.Field{}, expected: true, attr: "aggregatable"}, {commonField: common.Field{Aggregatable: &truthy}, expected: true, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &falsy}, expected: true, attr: "aggregatable"}, + {commonField: common.Field{Aggregatable: &falsy}, expected: false, attr: "aggregatable"}, {commonField: common.Field{Type: "keyword"}, expected: true, attr: "aggregatable"}, {commonField: common.Field{Aggregatable: &truthy, Type: "text"}, expected: false, attr: "aggregatable"}, {commonField: common.Field{Type: "text"}, expected: false, attr: "aggregatable"}, - // analyzed always set to false except for meta fields + // analyzed {commonField: common.Field{}, expected: false, attr: "analyzed"}, - {commonField: common.Field{Analyzed: &truthy}, expected: false, attr: "analyzed"}, + {commonField: common.Field{Analyzed: &truthy}, expected: true, attr: "analyzed"}, {commonField: common.Field{Analyzed: &falsy}, expected: false, attr: "analyzed"}, - // indexed always set to true except for meta fields - {commonField: common.Field{}, expected: true, attr: "indexed"}, - {commonField: common.Field{Index: &truthy}, expected: true, attr: "indexed"}, - {commonField: common.Field{Index: &falsy}, expected: true, attr: "indexed"}, - // doc_values always set to true except for meta fields {commonField: common.Field{}, expected: true, attr: "doc_values"}, {commonField: common.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, - {commonField: common.Field{DocValues: &falsy}, expected: true, attr: "doc_values"}, + {commonField: common.Field{DocValues: &falsy}, expected: false, attr: "doc_values"}, // scripted always set to false {commonField: common.Field{}, expected: false, attr: "scripted"}, From 166c5aed50659aea39b524b9af3ba5278ce4a347 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Tue, 3 Oct 2017 12:32:26 +0200 Subject: [PATCH 68/96] Support multiple format params. (#5299) Do not override other params if pattern is given. --- libbeat/common/field.go | 17 +++++++++++------ libbeat/kibana/transform.go | 22 ++++++++++++++++------ libbeat/kibana/transform_test.go | 28 +++++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/libbeat/common/field.go b/libbeat/common/field.go index a29f668b344..0a9118c7730 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -34,12 +34,17 @@ type Field struct { DocValues *bool `config:"doc_values"` // Kibana specific - Pattern string `config:"pattern"` - Analyzed *bool `config:"analyzed"` - Count int `config:"count"` - Searchable *bool `config:"searchable"` - Aggregatable *bool `config:"aggregatable"` - InputFormat string `config:"input_format"` + Analyzed *bool `config:"analyzed"` + Count int `config:"count"` + Searchable *bool `config:"searchable"` + Aggregatable *bool `config:"aggregatable"` + // Kibana field format specific + Pattern string `config:"pattern"` + InputFormat string `config:"input_format"` + OutputFormat string `config:"output_format"` + OutputPrecision string `config:"output_precision"` + LabelTemplate string `config:"label_template"` + UrlTemplate string `config:"url_template"` Path string } diff --git a/libbeat/kibana/transform.go b/libbeat/kibana/transform.go index 0972018bc78..10e38b3b1ad 100644 --- a/libbeat/kibana/transform.go +++ b/libbeat/kibana/transform.go @@ -113,13 +113,13 @@ func transformField(f common.Field) (common.MapStr, common.MapStr) { if f.Format != "" { format["id"] = f.Format - if f.InputFormat != "" { - format["params"] = common.MapStr{"inputFormat": f.InputFormat} - } - } - if f.Pattern != "" { - format["params"] = common.MapStr{"pattern": f.Pattern} + addFormatParam(&format, "inputFormat", f.InputFormat) + addFormatParam(&format, "outputFormat", f.OutputFormat) + addFormatParam(&format, "outputPrecision", f.OutputPrecision) + addFormatParam(&format, "labelTemplate", f.LabelTemplate) + addFormatParam(&format, "urlTemplate", f.UrlTemplate) } + addFormatParam(&format, "pattern", f.Pattern) } return field, format @@ -152,6 +152,16 @@ func getVal(valP *bool, def bool) bool { return def } +func addFormatParam(f *common.MapStr, key string, val string) { + if val == "" { + return + } + if (*f)["params"] == nil { + (*f)["params"] = common.MapStr{} + } + (*f)["params"].(common.MapStr)[key] = val +} + var ( typeMapping = map[string]string{ "half_float": "number", diff --git a/libbeat/kibana/transform_test.go b/libbeat/kibana/transform_test.go index 65856c78186..79e5d2c73ec 100644 --- a/libbeat/kibana/transform_test.go +++ b/libbeat/kibana/transform_test.go @@ -260,7 +260,8 @@ func TestTransformFieldFormatMap(t *testing.T) { "c": common.MapStr{ "id": "url", "params": common.MapStr{ - "pattern": "[^-]", + "pattern": "[^-]", + "inputFormat": "string", }, }, }, @@ -272,6 +273,31 @@ func TestTransformFieldFormatMap(t *testing.T) { }, expected: common.MapStr{}, }, + { + commonField: common.Field{ + Name: "c", + Format: "url", + Pattern: "[^-]", + InputFormat: "string", + OutputFormat: "float", + OutputPrecision: "3", + LabelTemplate: "lblT", + UrlTemplate: "urlT", + }, + expected: common.MapStr{ + "c": common.MapStr{ + "id": "url", + "params": common.MapStr{ + "pattern": "[^-]", + "inputFormat": "string", + "outputFormat": "float", + "outputPrecision": "3", + "labelTemplate": "lblT", + "urlTemplate": "urlT", + }, + }, + }, + }, } for idx, test := range tests { trans := NewTransformer("", "", common.Fields{test.commonField}) From 834c4c279db3d552df51ddf36d26cd5f511060b4 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Tue, 3 Oct 2017 13:21:34 +0200 Subject: [PATCH 69/96] Add scripts support. (#5298) Only allow painless as language. --- libbeat/common/field.go | 9 +++++---- libbeat/kibana/transform.go | 7 +++++++ libbeat/kibana/transform_test.go | 11 ++++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/libbeat/common/field.go b/libbeat/common/field.go index 0a9118c7730..12330b532c2 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -34,10 +34,11 @@ type Field struct { DocValues *bool `config:"doc_values"` // Kibana specific - Analyzed *bool `config:"analyzed"` - Count int `config:"count"` - Searchable *bool `config:"searchable"` - Aggregatable *bool `config:"aggregatable"` + Analyzed *bool `config:"analyzed"` + Count int `config:"count"` + Searchable *bool `config:"searchable"` + Aggregatable *bool `config:"aggregatable"` + Script string `config:"script"` // Kibana field format specific Pattern string `config:"pattern"` InputFormat string `config:"input_format"` diff --git a/libbeat/kibana/transform.go b/libbeat/kibana/transform.go index 10e38b3b1ad..2e5eb352a16 100644 --- a/libbeat/kibana/transform.go +++ b/libbeat/kibana/transform.go @@ -107,6 +107,13 @@ func transformField(f common.Field) (common.MapStr, common.MapStr) { field["aggregatable"] = false } + if f.Script != "" { + field["scripted"] = true + field["script"] = f.Script + field["lang"] = "painless" + field["doc_values"] = false + } + var format common.MapStr if f.Format != "" || f.Pattern != "" { format = common.MapStr{} diff --git a/libbeat/kibana/transform_test.go b/libbeat/kibana/transform_test.go index 79e5d2c73ec..7a720ac2748 100644 --- a/libbeat/kibana/transform_test.go +++ b/libbeat/kibana/transform_test.go @@ -185,9 +185,18 @@ func TestTransformMisc(t *testing.T) { {commonField: common.Field{}, expected: true, attr: "doc_values"}, {commonField: common.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, {commonField: common.Field{DocValues: &falsy}, expected: false, attr: "doc_values"}, + {commonField: common.Field{Script: "doc[]"}, expected: false, attr: "doc_values"}, + {commonField: common.Field{DocValues: &truthy, Script: "doc[]"}, expected: false, attr: "doc_values"}, - // scripted always set to false + // script, scripted {commonField: common.Field{}, expected: false, attr: "scripted"}, + {commonField: common.Field{}, expected: nil, attr: "script"}, + {commonField: common.Field{Script: "doc[]"}, expected: true, attr: "scripted"}, + {commonField: common.Field{Script: "doc[]"}, expected: "doc[]", attr: "script"}, + + // language + {commonField: common.Field{}, expected: nil, attr: "lang"}, + {commonField: common.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, } for idx, test := range tests { trans := NewTransformer("", "", common.Fields{test.commonField}) From 91bea8e62b2100d7211fb3601c840ff22120aec1 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 3 Oct 2017 14:33:27 +0200 Subject: [PATCH 70/96] Run make misspell (#5300) * Exclude .json files from misspell as Kibana files contain `seperate_axis` which is misspelled but part of Kibana and should not be fixed. * Fix Transfered to Transferred manually in Kibana files --- Makefile | 3 ++- filebeat/docs/fields.asciidoc | 4 ++-- filebeat/fileset/modules.go | 2 +- filebeat/harvester/reader/multiline_test.go | 2 +- filebeat/module/postgresql/log/_meta/fields.yml | 4 ++-- filebeat/prospector/log/harvester.go | 2 +- filebeat/prospector/log/prospector.go | 2 +- filebeat/prospector/registry.go | 2 +- heartbeat/monitors/active/dialchain/net.go | 4 ++-- heartbeat/monitors/active/dialchain/util.go | 4 ++-- libbeat/beat/pipeline.go | 2 +- libbeat/cfgfile/reload.go | 2 +- libbeat/cmd/root.go | 6 +++--- libbeat/common/match/optimize.go | 2 +- libbeat/dashboards/es_loader.go | 2 +- libbeat/docs/command-reference.asciidoc | 2 +- libbeat/docs/outputconfig.asciidoc | 4 ++-- libbeat/kibana/testdata/extensive/fields.yml | 4 ++-- libbeat/monitoring/report/elasticsearch/elasticsearch.go | 2 +- libbeat/outputs/outputs.go | 2 +- libbeat/publisher/TODO.txt | 2 +- libbeat/publisher/pipeline/acker.go | 4 ++-- libbeat/publisher/pipeline/controller.go | 2 +- libbeat/publisher/queue/memqueue/consume.go | 2 +- libbeat/publisher/queue/memqueue/eventloop.go | 4 ++-- metricbeat/docs/fields.asciidoc | 5 ++++- metricbeat/module/kubernetes/event/_meta/fields.yml | 2 +- metricbeat/module/prometheus/collector/data.go | 6 +++--- .../visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json | 2 +- .../visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json | 2 +- .../kibana/default/dashboard/Metricbeat-host-overview.json | 6 +++--- .../default/dashboard/Metricbeat-system-overview.json | 6 +++--- metricbeat/module/vsphere/virtualmachine/_meta/fields.yml | 2 +- 33 files changed, 53 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index b76016bb86a..19b7d92a2cf 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,8 @@ check: python-env .PHONY: misspell misspell: go get github.com/client9/misspell - $(FIND) -name '*' -exec misspell -w {} \; + # Ignore Kibana files (.json) + $(FIND) -not -path "*.json" -name '*' -exec misspell -w {} \; .PHONY: fmt fmt: python-env diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 0b677ee0073..a098815b6ed 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1406,7 +1406,7 @@ The timezone of timestamp. type: long -Processs id +Process id [float] @@ -1444,7 +1444,7 @@ Duration of a query. example: SELECT * FROM users; -Query statment. +Query statement. [float] === `postgresql.log.message` diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go index 398997df139..82dbb298785 100644 --- a/filebeat/fileset/modules.go +++ b/filebeat/fileset/modules.go @@ -443,7 +443,7 @@ func interpretError(initialErr error, body []byte) error { return fmt.Errorf("couldn't load pipeline: %v. Response body: %s", initialErr, body) } -// LoadML loads the machine-learning configurations into Elasticsearch, if Xpack is avaiable +// LoadML loads the machine-learning configurations into Elasticsearch, if Xpack is available func (reg *ModuleRegistry) LoadML(esClient PipelineLoader) error { haveXpack, err := mlimporter.HaveXpackML(esClient) if err != nil { diff --git a/filebeat/harvester/reader/multiline_test.go b/filebeat/harvester/reader/multiline_test.go index 018983a0435..e3d628eb783 100644 --- a/filebeat/harvester/reader/multiline_test.go +++ b/filebeat/harvester/reader/multiline_test.go @@ -94,7 +94,7 @@ func TestMultilineAfterNegateOKFlushPattern(t *testing.T) { }, 3, "EventStart\nEventId: 1\nEventEnd\n", - "OtherThingInBetween\n", // this should be a seperate event.. + "OtherThingInBetween\n", // this should be a separate event.. "EventStart\nEventId: 2\nEventEnd\n", ) } diff --git a/filebeat/module/postgresql/log/_meta/fields.yml b/filebeat/module/postgresql/log/_meta/fields.yml index 06e01ca7862..dba877b66f2 100644 --- a/filebeat/module/postgresql/log/_meta/fields.yml +++ b/filebeat/module/postgresql/log/_meta/fields.yml @@ -12,7 +12,7 @@ - name: thread_id type: long description: > - Processs id + Process id - name: user example: "admin" description: @@ -33,7 +33,7 @@ - name: query example: "SELECT * FROM users;" description: - Query statment. + Query statement. - name: message type: text description: > diff --git a/filebeat/prospector/log/harvester.go b/filebeat/prospector/log/harvester.go index a163e3cd5b0..ae5e032902d 100644 --- a/filebeat/prospector/log/harvester.go +++ b/filebeat/prospector/log/harvester.go @@ -156,7 +156,7 @@ func (h *Harvester) Setup() error { func (h *Harvester) Run() error { // This is to make sure a harvester is not started anymore if stop was already // called before the harvester was started. The waitgroup is not incremented afterwards - // as otherwise it could happend that between checking for the close channel and incrementing + // as otherwise it could happened that between checking for the close channel and incrementing // the waitgroup, the harvester could be stopped. h.stopWg.Add(1) select { diff --git a/filebeat/prospector/log/prospector.go b/filebeat/prospector/log/prospector.go index e0331c28c0b..a641aae24ba 100644 --- a/filebeat/prospector/log/prospector.go +++ b/filebeat/prospector/log/prospector.go @@ -569,7 +569,7 @@ func (p *Prospector) isCleanInactive(state file.State) bool { // createHarvester creates a new harvester instance from the given state func (p *Prospector) createHarvester(state file.State) (*Harvester, error) { - // Each wraps the outlet, for closing the outlet individualy + // Each wraps the outlet, for closing the outlet individually outlet := channel.SubOutlet(p.outlet) h, err := NewHarvester( p.cfg, diff --git a/filebeat/prospector/registry.go b/filebeat/prospector/registry.go index 8585cef483f..b26d0a01d64 100644 --- a/filebeat/prospector/registry.go +++ b/filebeat/prospector/registry.go @@ -32,7 +32,7 @@ func Register(name string, factory Factory) error { } registry[name] = factory - logp.Info("Succesfully registered prospector") + logp.Info("Successfully registered prospector") return nil } diff --git a/heartbeat/monitors/active/dialchain/net.go b/heartbeat/monitors/active/dialchain/net.go index 295ccf68f9c..6f0018d62cd 100644 --- a/heartbeat/monitors/active/dialchain/net.go +++ b/heartbeat/monitors/active/dialchain/net.go @@ -13,7 +13,7 @@ import ( // TCPDialer creates a new NetDialer with constant event fields and default // connection timeout. -// The fields parameter holds additonal constants to be added to the final event +// The fields parameter holds additional constants to be added to the final event // structure. // // The dialer will update the active events with: @@ -30,7 +30,7 @@ func TCPDialer(to time.Duration) NetDialer { // UDPDialer creates a new NetDialer with constant event fields and default // connection timeout. -// The fields parameter holds additonal constants to be added to the final event +// The fields parameter holds additional constants to be added to the final event // structure. // // The dialer will update the active events with: diff --git a/heartbeat/monitors/active/dialchain/util.go b/heartbeat/monitors/active/dialchain/util.go index b5ea4885dd8..d0e9322ff42 100644 --- a/heartbeat/monitors/active/dialchain/util.go +++ b/heartbeat/monitors/active/dialchain/util.go @@ -32,14 +32,14 @@ func ConstAddrLayer(address string) Layer { } // MakeConstAddrLayer always passes the same address to the original Layer. -// This is usefull if a lookup did return multiple IPs for the same hostname, +// This is useful if a lookup did return multiple IPs for the same hostname, // but the IP use to connect shall be fixed. func MakeConstAddrLayer(addr string, origLayer Layer) Layer { return withLayerDialer(origLayer, constAddr(addr)) } // MakeConstAddrDialer always passes the same address to the original NetDialer. -// This is usefull if a lookup did return multiple IPs for the same hostname, +// This is useful if a lookup did return multiple IPs for the same hostname, // but the IP use to connect shall be fixed. func MakeConstAddrDialer(addr string, origNet NetDialer) NetDialer { return withNetDialer(origNet, constAddr(addr)) diff --git a/libbeat/beat/pipeline.go b/libbeat/beat/pipeline.go index 2b5bf5389c1..d4bf495c8a8 100644 --- a/libbeat/beat/pipeline.go +++ b/libbeat/beat/pipeline.go @@ -120,7 +120,7 @@ const ( // DropIfFull drops an event to be send if the pipeline is currently full. // This ensures a beats internals can continue processing if the pipeline has - // filled up. Usefull if an event stream must be processed to keep internal + // filled up. Useful if an event stream must be processed to keep internal // state up-to-date. DropIfFull ) diff --git a/libbeat/cfgfile/reload.go b/libbeat/cfgfile/reload.go index b2f0e5ec73f..7601e3d9dd6 100644 --- a/libbeat/cfgfile/reload.go +++ b/libbeat/cfgfile/reload.go @@ -134,7 +134,7 @@ func (rl *Reloader) Run(runnerFactory RunnerFactory) { gw := NewGlobWatcher(rl.path) - // If reloading is disable, config files should be loaded immidiately + // If reloading is disable, config files should be loaded immediately if !rl.config.Reload.Enabled { rl.config.Reload.Period = 0 } diff --git a/libbeat/cmd/root.go b/libbeat/cmd/root.go index 31d8169c9b7..9bc0a9b226d 100644 --- a/libbeat/cmd/root.go +++ b/libbeat/cmd/root.go @@ -33,14 +33,14 @@ type BeatsRootCmd struct { } // GenRootCmd returns the root command to use for your beat. It takes -// beat name as paramter, and also run command, which will be called if no args are +// beat name as parameter, and also run command, which will be called if no args are // given (for backwards compatibility) func GenRootCmd(name, version string, beatCreator beat.Creator) *BeatsRootCmd { return GenRootCmdWithRunFlags(name, version, beatCreator, nil) } // GenRootCmdWithRunFlags returns the root command to use for your beat. It takes -// beat name as paramter, and also run command, which will be called if no args are +// beat name as parameter, and also run command, which will be called if no args are // given (for backwards compatibility). runFlags parameter must the flagset used by // run command func GenRootCmdWithRunFlags(name, version string, beatCreator beat.Creator, runFlags *pflag.FlagSet) *BeatsRootCmd { @@ -61,7 +61,7 @@ func GenRootCmdWithIndexPrefixWithRunFlags(name, indexPrefix, version string, be // Root command is an alias for run rootCmd.Run = rootCmd.RunCmd.Run - // Persistent flags, common accross all subcommands + // Persistent flags, common across all subcommands rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("E")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("c")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("d")) diff --git a/libbeat/common/match/optimize.go b/libbeat/common/match/optimize.go index 65e77062dff..aa757c6a420 100644 --- a/libbeat/common/match/optimize.go +++ b/libbeat/common/match/optimize.go @@ -218,7 +218,7 @@ func flattenRepetition(r *syntax.Regexp) (bool, *syntax.Regexp) { sub = sub[:1] inRepetition = true - // create flattened regex repetition mulitplying count + // create flattened regex repetition multiplying count // if nexted expression is also a repetition if s := sub[0]; isConcatRepetition(s) { count := len(s.Sub) * len(r.Sub) diff --git a/libbeat/dashboards/es_loader.go b/libbeat/dashboards/es_loader.go index 8c2784138fc..4f63821b397 100644 --- a/libbeat/dashboards/es_loader.go +++ b/libbeat/dashboards/es_loader.go @@ -195,7 +195,7 @@ func (loader ElasticsearchLoader) importVisualization(file string) error { var vizContent common.MapStr err = json.Unmarshal(reader, &vizContent) if err != nil { - return fmt.Errorf("fail to unmarshal vizualization content %s: %v", file, err) + return fmt.Errorf("fail to unmarshal visualization content %s: %v", file, err) } if loader.config.Index != "" { diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index 7dbb0d3f3a2..d777ab89d8d 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -96,7 +96,7 @@ command exports the configuration that's defined in the specified file. [[template-subcommand]] *`template`*:: Exports the index template to stdout. You can specify the `--es.version` and -`--index` flags to futher define what gets exported. +`--index` flags to further define what gets exported. *FLAGS* diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 01f4df07368..cd81954c5eb 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -533,7 +533,7 @@ spooler size. ===== `slow_start` -If enabled only a subset of events in a batch of events is transfered per transaction. +If enabled only a subset of events in a batch of events is transferred per transaction. The number of events to be sent increases up to `bulk_max_size` if no error is encountered. On error the number of events per transaction is reduced again. @@ -1067,7 +1067,7 @@ setting does not affect how events are published. Setting `bulk_max_size` to 0 disables buffering in libbeat. [[configuration-output-codec]] -=== Configure the ouput codec +=== Configure the output codec ++++ Output codec diff --git a/libbeat/kibana/testdata/extensive/fields.yml b/libbeat/kibana/testdata/extensive/fields.yml index f8a8c7d835e..9c6a4e96032 100644 --- a/libbeat/kibana/testdata/extensive/fields.yml +++ b/libbeat/kibana/testdata/extensive/fields.yml @@ -2873,7 +2873,7 @@ - name: count type: long description: > - Count field records the number of times the particular event has occured + Count field records the number of times the particular event has occurred fields: - name: timestamp type: group @@ -6745,7 +6745,7 @@ - name: custom_fields type: object object_type: keyword - descripton: > + description: > Custom fields - key: windows diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index dab27f87fb7..b0dcdd47350 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -58,7 +58,7 @@ func makeReporter(beat beat.Info, cfg *common.Config) (report.Reporter, error) { return nil, err } - // check endpoint availablity on startup only every 30 seconds + // check endpoint availability on startup only every 30 seconds checkRetry := 30 * time.Second windowSize := config.BulkMaxSize - 1 if windowSize <= 0 { diff --git a/libbeat/outputs/outputs.go b/libbeat/outputs/outputs.go index 26d2a5854e3..eead5b6611a 100644 --- a/libbeat/outputs/outputs.go +++ b/libbeat/outputs/outputs.go @@ -20,7 +20,7 @@ type Client interface { Publish(publisher.Batch) error } -// NetworkClient defines the required client capabilites for network based +// NetworkClient defines the required client capabilities for network based // outputs, that must be reconnectable. type NetworkClient interface { Client diff --git a/libbeat/publisher/TODO.txt b/libbeat/publisher/TODO.txt index 513ec78c3f9..74901c8382a 100644 --- a/libbeat/publisher/TODO.txt +++ b/libbeat/publisher/TODO.txt @@ -4,6 +4,6 @@ followup: - elasticsearch output update: - replace json reader with go-structform on struct - move package pipeline go files to libbeat/publisher if possible -- '@metadata' accessor from withing fmtstr +- '@metadata' accessor from within fmtstr - remove publisher/bc package - register some pipeline metrics for consumption diff --git a/libbeat/publisher/pipeline/acker.go b/libbeat/publisher/pipeline/acker.go index f7e0891fd12..231d85e4f77 100644 --- a/libbeat/publisher/pipeline/acker.go +++ b/libbeat/publisher/pipeline/acker.go @@ -61,7 +61,7 @@ func (a *countACK) ackEvents(n int) { // gapCountACK returns event ACKs to the producer, taking account for dropped events. // Events being dropped by processors will always be ACKed with the last batch ACKed // by the broker. This way clients waiting for ACKs can expect all processed -// events being alwyas ACKed. +// events being always ACKed. type gapCountACK struct { pipeline *Pipeline @@ -134,7 +134,7 @@ func (a *gapCountACK) ackLoop() { } case <-drop: - // TODO: accumulate mulitple drop events + flush count with timer + // TODO: accumulate multiple drop events + flush count with timer a.fn(1, 0) } } diff --git a/libbeat/publisher/pipeline/controller.go b/libbeat/publisher/pipeline/controller.go index bc92b4da91f..7595a0cd882 100644 --- a/libbeat/publisher/pipeline/controller.go +++ b/libbeat/publisher/pipeline/controller.go @@ -6,7 +6,7 @@ import ( "github.com/elastic/beats/libbeat/publisher/queue" ) -// outputController manages the pipelines output capabilites, like: +// outputController manages the pipelines output capabilities, like: // - start // - stop // - reload diff --git a/libbeat/publisher/queue/memqueue/consume.go b/libbeat/publisher/queue/memqueue/consume.go index 0a9159bca62..529c7b1bc27 100644 --- a/libbeat/publisher/queue/memqueue/consume.go +++ b/libbeat/publisher/queue/memqueue/consume.go @@ -59,7 +59,7 @@ func (c *consumer) Get(sz int) (queue.Batch, error) { return nil, io.EOF } - // if request has been send, we do have to wait for a reponse + // if request has been send, we do have to wait for a response resp := <-c.resp ack := resp.ack diff --git a/libbeat/publisher/queue/memqueue/eventloop.go b/libbeat/publisher/queue/memqueue/eventloop.go index fd3edca15c3..e3f6a5291d6 100644 --- a/libbeat/publisher/queue/memqueue/eventloop.go +++ b/libbeat/publisher/queue/memqueue/eventloop.go @@ -242,7 +242,7 @@ func (l *directEventLoop) processACK(lst chanList, N int) { count := (st.seq - st.state.lastACK) if count == 0 || count > math.MaxUint32/2 { // seq number comparison did underflow. This happens only if st.seq has - // allready been acknowledged + // already been acknowledged // log.Debug("seq number already acked: ", st.seq) st.state = nil @@ -508,7 +508,7 @@ func (l *bufferingEventLoop) processACK(lst chanList, N int) { count := st.seq - st.state.lastACK if count == 0 || count > math.MaxUint32/2 { // seq number comparison did underflow. This happens only if st.seq has - // allready been acknowledged + // already been acknowledged // log.Debug("seq number already acked: ", st.seq) st.state = nil diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2301923d53b..608a4013abd 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4767,7 +4767,7 @@ The Kubernetes events metricset collects events that are generated by objects ru type: long -Count field records the number of times the particular event has occured +Count field records the number of times the particular event has occurred [float] @@ -10972,6 +10972,9 @@ Free Memory of Guest in bytes type: object +Custom fields + + [[exported-fields-windows]] == Windows fields diff --git a/metricbeat/module/kubernetes/event/_meta/fields.yml b/metricbeat/module/kubernetes/event/_meta/fields.yml index 940109dd206..010c22559a6 100644 --- a/metricbeat/module/kubernetes/event/_meta/fields.yml +++ b/metricbeat/module/kubernetes/event/_meta/fields.yml @@ -7,7 +7,7 @@ - name: count type: long description: > - Count field records the number of times the particular event has occured + Count field records the number of times the particular event has occurred fields: - name: timestamp type: group diff --git a/metricbeat/module/prometheus/collector/data.go b/metricbeat/module/prometheus/collector/data.go index de7e33986e2..c51b338e3b3 100644 --- a/metricbeat/module/prometheus/collector/data.go +++ b/metricbeat/module/prometheus/collector/data.go @@ -46,9 +46,9 @@ func GetPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { value["value"] = int64(counter.GetValue()) } - guage := metric.GetGauge() - if guage != nil { - value["value"] = guage.GetValue() + gauge := metric.GetGauge() + if gauge != nil { + value["value"] = gauge.GetValue() } summary := metric.GetSummary() diff --git a/metricbeat/module/system/_meta/kibana/5.x/visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json b/metricbeat/module/system/_meta/kibana/5.x/visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json index 1853816834c..f2536b1f3c6 100644 --- a/metricbeat/module/system/_meta/kibana/5.x/visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json +++ b/metricbeat/module/system/_meta/kibana/5.x/visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json @@ -1,5 +1,5 @@ { - "visState": "{\"title\":\"Metricbeat: Outbound Traffic Metric\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Outbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"a1737470-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transfered\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\"},\"aggs\":[],\"listeners\":{}}", + "visState": "{\"title\":\"Metricbeat: Outbound Traffic Metric\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Outbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"a1737470-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transferred\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\"},\"aggs\":[],\"listeners\":{}}", "description": "", "title": "Metricbeat: Outbound Traffic Metric", "uiStateJSON": "{}", diff --git a/metricbeat/module/system/_meta/kibana/5.x/visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json b/metricbeat/module/system/_meta/kibana/5.x/visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json index 695294913f2..689e981de83 100644 --- a/metricbeat/module/system/_meta/kibana/5.x/visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json +++ b/metricbeat/module/system/_meta/kibana/5.x/visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json @@ -1,5 +1,5 @@ { - "visState": "{\"title\":\"Metricbeat: Inbound Traffic Metric\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Inbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"c40e18f0-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transfered\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\"},\"aggs\":[],\"listeners\":{}}", + "visState": "{\"title\":\"Metricbeat: Inbound Traffic Metric\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Inbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"c40e18f0-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transferred\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\"},\"aggs\":[],\"listeners\":{}}", "description": "", "title": "Metricbeat: Inbound Traffic Metric", "uiStateJSON": "{}", diff --git a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-host-overview.json b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-host-overview.json index a94b6014269..c3fae5ecd76 100644 --- a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-host-overview.json +++ b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-host-overview.json @@ -174,7 +174,7 @@ "title": "Inbound Traffic [Metricbeat System]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Inbound Traffic [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Inbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"c40e18f0-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transfered\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\",\"show_grid\":1},\"aggs\":[]}" + "visState": "{\"title\":\"Inbound Traffic [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Inbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"c40e18f0-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transferred\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\",\"show_grid\":1},\"aggs\":[]}" }, "id": "522ee670-1b92-11e7-bec4-a5e9ec5cab8b", "type": "visualization", @@ -189,7 +189,7 @@ "title": "Outbound Traffic [Metricbeat System]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Outbound Traffic [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Outbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"a1737470-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transfered\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\",\"show_grid\":1},\"aggs\":[]}" + "visState": "{\"title\":\"Outbound Traffic [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Outbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"a1737470-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transferred\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\",\"show_grid\":1},\"aggs\":[]}" }, "id": "1aae9140-1b93-11e7-8ada-3df93aab833e", "type": "visualization", @@ -357,4 +357,4 @@ } ], "version": "6.0.0-beta1-SNAPSHOT" -} \ No newline at end of file +} diff --git a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-system-overview.json b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-system-overview.json index 828c0a9447e..e1f7122bf5a 100644 --- a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-system-overview.json +++ b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-system-overview.json @@ -84,7 +84,7 @@ "title": "Inbound Traffic [Metricbeat System]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Inbound Traffic [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Inbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"c40e18f0-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transfered\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\",\"show_grid\":1},\"aggs\":[]}" + "visState": "{\"title\":\"Inbound Traffic [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Inbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"c40e18f0-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transferred\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\",\"show_grid\":1},\"aggs\":[]}" }, "id": "522ee670-1b92-11e7-bec4-a5e9ec5cab8b", "type": "visualization", @@ -99,7 +99,7 @@ "title": "Outbound Traffic [Metricbeat System]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Outbound Traffic [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Outbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"a1737470-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transfered\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\",\"show_grid\":1},\"aggs\":[]}" + "visState": "{\"title\":\"Outbound Traffic [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Outbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"},{\"id\":\"a1737470-2c55-11e7-a0ad-277ce466684d\",\"type\":\"series_agg\",\"function\":\"sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\",\"terms_field\":\"system.network.name\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transferred\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"series_agg\",\"function\":\"overall_sum\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"value_template\":\"{{value}}\",\"terms_field\":\"system.network.name\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\",\"show_grid\":1},\"aggs\":[]}" }, "id": "1aae9140-1b93-11e7-8ada-3df93aab833e", "type": "visualization", @@ -170,4 +170,4 @@ } ], "version": "6.0.0-beta1-SNAPSHOT" -} \ No newline at end of file +} diff --git a/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml b/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml index 9d2d206804a..2595182b349 100644 --- a/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml +++ b/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml @@ -38,5 +38,5 @@ - name: custom_fields type: object object_type: keyword - descripton: > + description: > Custom fields From 806ee8300c9809dfeddf8900088b37ba16525438 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 3 Oct 2017 17:36:43 +0200 Subject: [PATCH 71/96] Make cleanup more reliable (#5303) Currently some jenkins job do not properly cleanup. This change should make the cleanup more reliable by separating the fix-permissions command and and make sure it also runs if stop-environment was not successful. --- dev-tools/jenkins_ci.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-tools/jenkins_ci.sh b/dev-tools/jenkins_ci.sh index 087d8ae50b1..f89272ad437 100755 --- a/dev-tools/jenkins_ci.sh +++ b/dev-tools/jenkins_ci.sh @@ -28,7 +28,8 @@ export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env" cleanup() { echo "Running cleanup..." rm -rf $TEMP_PYTHON_ENV - make stop-environment fix-permissions + make stop-environment || true + make fix-permissions || true echo "Killing all running containers..." docker ps -q | xargs -r docker kill || true echo "Cleaning stopped docker containers and dangling images/networks/volumes..." @@ -39,4 +40,4 @@ trap cleanup EXIT rm -rf ${GOPATH}/pkg cd ${beat} -RACE_DETECTOR=1 make clean check testsuite \ No newline at end of file +RACE_DETECTOR=1 make clean check testsuite From 6b85d185af8926df80cfe9c71314ea36787b94dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 3 Oct 2017 19:32:50 +0200 Subject: [PATCH 72/96] Fix data race accessing watched containers (#5307) --- CHANGELOG.asciidoc | 1 + libbeat/processors/add_docker_metadata/watcher.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 097e56781f7..915e6772567 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -14,6 +14,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di *Affecting all Beats* - Fix kubernetes matcher registry lookup. {pull}5159[5159] +- Fix data race accessing watched containers. {issue}5147[5147] *Auditbeat* diff --git a/libbeat/processors/add_docker_metadata/watcher.go b/libbeat/processors/add_docker_metadata/watcher.go index 4cfee779bc9..91ea5fddb4a 100644 --- a/libbeat/processors/add_docker_metadata/watcher.go +++ b/libbeat/processors/add_docker_metadata/watcher.go @@ -108,10 +108,11 @@ func NewWatcherWithClient(client Client, cleanupTimeout time.Duration) (*watcher func (w *watcher) Container(ID string) *Container { w.RLock() container := w.containers[ID] + _, ok := w.deleted[ID] w.RUnlock() // Update last access time if it's deleted - if _, ok := w.deleted[ID]; ok { + if ok { w.Lock() w.deleted[ID] = time.Now() w.Unlock() From 6e07d0b55a8975e44f1018a07688a6683bdeac2f Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Tue, 3 Oct 2017 20:57:07 +0200 Subject: [PATCH 73/96] Add Pre-Commit hook. (#5305) --- Makefile | 6 ++++++ script/pre_commit.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100755 script/pre_commit.sh diff --git a/Makefile b/Makefile index 19b7d92a2cf..60bc0a45ff3 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,12 @@ REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection. # Also it builds the docs and the generators + +.PHONY: setup-commit-hook +setup-commit-hook: + @cp script/pre_commit.sh .git/hooks/pre-commit + @chmod 751 .git/hooks/pre-commit + .PHONY: testsuite testsuite: @$(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;) diff --git a/script/pre_commit.sh b/script/pre_commit.sh new file mode 100755 index 00000000000..bc79b868d81 --- /dev/null +++ b/script/pre_commit.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -e + +echo "-- pre commit hook running" +#return if no files staged for commit +staged_files=$(git diff --name-only --cached) +[ -z "$staged_files" ] && exit 0 + +#run make cmds and check whether +#- unstaged files have changed +#- make cmd fails +unstaged_files=$(git diff --name-only) + +echo "---- lint" +make lint + +echo "---- format" +make fmt + +echo "---- misspell" +make misspell + +unstaged_files_after=$(git diff --name-only) +if [ "$unstaged_files" == "$unstaged_files_after" ] ; then + exit 0 +fi; +echo "Pre-Commit hook has failed, see changed files." +exit 1 From 16be171c66fa159404c8101c398ebe41b5744c4d Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 4 Oct 2017 11:10:15 +0200 Subject: [PATCH 74/96] Increase Kibana timeout for tests (#5316) Kibana can take quite some time to start in a slow / busy environment. Timeout is now increased to 5 minutes. --- metricbeat/tests/system/test_kibana.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py index 18d212527f4..7d36d97f4d8 100644 --- a/metricbeat/tests/system/test_kibana.py +++ b/metricbeat/tests/system/test_kibana.py @@ -8,7 +8,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['elasticsearch', 'kibana'] - COMPOSE_TIMEOUT = 120 + COMPOSE_TIMEOUT = 300 @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_status(self): From 9b52b392e7de0aaee74ecdde3d31354693343e08 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 4 Oct 2017 15:02:29 +0200 Subject: [PATCH 75/96] Fix flaky Filebeat test on Windows (#5314) It seems as the file was persisted and then beats started sometime the timestamp of the file was identical to now so ignore_older did not apply. Now waiting 1s for the beat to start should solve this issue. --- filebeat/tests/system/test_crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index 81b0cce4228..13008bf9e18 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -470,6 +470,9 @@ def test_tail_files(self): f.write("hello world 2\n") f.flush() + # Sleep 1 second to make sure the file is persisted on disk and timestamp is in the past + time.sleep(1) + filebeat = self.start_beat() self.wait_until( lambda: self.log_contains( From 4cb87141eca1f7072f0d2c0ea73ab31ed12f62cb Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Wed, 4 Oct 2017 15:48:54 +0200 Subject: [PATCH 76/96] Fix OutputPrecision to be int instead of string. (#5318) --- libbeat/common/field.go | 2 +- .../kibana/index_pattern_generator_test.go | 2 +- libbeat/kibana/transform.go | 24 +++++++++++++------ libbeat/kibana/transform_test.go | 5 ++-- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/libbeat/common/field.go b/libbeat/common/field.go index 12330b532c2..bda663e6686 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -43,7 +43,7 @@ type Field struct { Pattern string `config:"pattern"` InputFormat string `config:"input_format"` OutputFormat string `config:"output_format"` - OutputPrecision string `config:"output_precision"` + OutputPrecision *int `config:"output_precision"` LabelTemplate string `config:"label_template"` UrlTemplate string `config:"url_template"` diff --git a/libbeat/kibana/index_pattern_generator_test.go b/libbeat/kibana/index_pattern_generator_test.go index a450315b57f..093d8b3efdb 100644 --- a/libbeat/kibana/index_pattern_generator_test.go +++ b/libbeat/kibana/index_pattern_generator_test.go @@ -172,7 +172,7 @@ func testGenerate(t *testing.T, beatDir string, tests []map[string]string) { for _, e := range fieldsExisting { idx := find(fieldsCreated, e["name"].(string)) assert.NotEqual(t, -1, idx) - assert.Equal(t, fieldsCreated[idx], e) + assert.Equal(t, e, fieldsCreated[idx]) } } } diff --git a/libbeat/kibana/transform.go b/libbeat/kibana/transform.go index 2e5eb352a16..2b668aeae33 100644 --- a/libbeat/kibana/transform.go +++ b/libbeat/kibana/transform.go @@ -120,13 +120,13 @@ func transformField(f common.Field) (common.MapStr, common.MapStr) { if f.Format != "" { format["id"] = f.Format - addFormatParam(&format, "inputFormat", f.InputFormat) - addFormatParam(&format, "outputFormat", f.OutputFormat) - addFormatParam(&format, "outputPrecision", f.OutputPrecision) - addFormatParam(&format, "labelTemplate", f.LabelTemplate) - addFormatParam(&format, "urlTemplate", f.UrlTemplate) + addStringFormatParam(&format, "inputFormat", f.InputFormat) + addStringFormatParam(&format, "outputFormat", f.OutputFormat) + addIntPFormatParam(&format, "outputPrecision", f.OutputPrecision) + addStringFormatParam(&format, "labelTemplate", f.LabelTemplate) + addStringFormatParam(&format, "urlTemplate", f.UrlTemplate) } - addFormatParam(&format, "pattern", f.Pattern) + addStringFormatParam(&format, "pattern", f.Pattern) } return field, format @@ -159,7 +159,7 @@ func getVal(valP *bool, def bool) bool { return def } -func addFormatParam(f *common.MapStr, key string, val string) { +func addStringFormatParam(f *common.MapStr, key string, val string) { if val == "" { return } @@ -169,6 +169,16 @@ func addFormatParam(f *common.MapStr, key string, val string) { (*f)["params"].(common.MapStr)[key] = val } +func addIntPFormatParam(f *common.MapStr, key string, valP *int) { + if valP == nil { + return + } + if (*f)["params"] == nil { + (*f)["params"] = common.MapStr{} + } + (*f)["params"].(common.MapStr)[key] = *valP +} + var ( typeMapping = map[string]string{ "half_float": "number", diff --git a/libbeat/kibana/transform_test.go b/libbeat/kibana/transform_test.go index 7a720ac2748..6c314690049 100644 --- a/libbeat/kibana/transform_test.go +++ b/libbeat/kibana/transform_test.go @@ -207,6 +207,7 @@ func TestTransformMisc(t *testing.T) { } func TestTransformFieldFormatMap(t *testing.T) { + precision := 3 tests := []struct { commonField common.Field expected common.MapStr @@ -289,7 +290,7 @@ func TestTransformFieldFormatMap(t *testing.T) { Pattern: "[^-]", InputFormat: "string", OutputFormat: "float", - OutputPrecision: "3", + OutputPrecision: &precision, LabelTemplate: "lblT", UrlTemplate: "urlT", }, @@ -300,7 +301,7 @@ func TestTransformFieldFormatMap(t *testing.T) { "pattern": "[^-]", "inputFormat": "string", "outputFormat": "float", - "outputPrecision": "3", + "outputPrecision": 3, "labelTemplate": "lblT", "urlTemplate": "urlT", }, From d4633ad038976fd7b7f79c9bd0d9097199f0244f Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Wed, 4 Oct 2017 22:44:31 -0700 Subject: [PATCH 77/96] Add Collectbeat to community beats (#5327) --- CHANGELOG.asciidoc | 1 + libbeat/docs/communitybeats.asciidoc | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 915e6772567..dd9e357c924 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -78,6 +78,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Fix reloader error message to only print on actual error {pull}5066[5066] - Enable flush timeout by default. {pull}5150[5150] - Add @metadata.version to events send to Logstash. {pull}5166[5166] +- Add Collectbeat to community beats. {pull}5327[5327] *Auditbeat* diff --git a/libbeat/docs/communitybeats.asciidoc b/libbeat/docs/communitybeats.asciidoc index 6ed0aeff937..9c566449eec 100644 --- a/libbeat/docs/communitybeats.asciidoc +++ b/libbeat/docs/communitybeats.asciidoc @@ -29,6 +29,7 @@ https://github.com/jarl-tornroos/cloudfrontbeat[cloudfrontbeat]:: Reads log even https://github.com/aidan-/cloudtrailbeat[cloudtrailbeat]:: Reads events from Amazon Web Services' https://aws.amazon.com/cloudtrail/[CloudTrail]. https://github.com/narmitech/cloudwatchmetricbeat[cloudwatchmetricbeat]:: A beat for Amazon Web Services' https://aws.amazon.com/cloudwatch/details/#other-aws-resource-monitoring[CloudWatch Metrics]. https://github.com/e-travel/cloudwatchlogsbeat[cloudwatchlogsbeat]:: Reads log events from Amazon Web Services' https://aws.amazon.com/cloudwatch/details/#log-monitoring[CloudWatch Logs]. +https://github.com/eBay/collectbeat[collectbeat]:: Adds discovery on top of Filebeat and Metricbeat in environments like Kubernetes. https://github.com/raboof/connbeat[connbeat]:: Exposes metadata about TCP connections. https://github.com/Pravoru/consulbeat[consulbeat]:: Reads services health checks from consul and pushes them to Elastic. https://github.com/Ingensi/dockbeat[dockbeat]:: Reads Docker container From 396cff7daa9f24ca5f4a6bb01362fde61c6188ab Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 5 Oct 2017 09:09:57 +0200 Subject: [PATCH 78/96] Deprecate config_dir (Filebeat) and refactor docs (#5321) This deprecates the config_dir option in Filebeat, as it's functionality is covered by `filebeat.config.prospectors`. There's a bigger refactoring in the docs, to reflect that the `filebeat.config.*` and `metricbeat.config.*` options can be used for both including external files and live reloading. Only the live reloading part stays in Beta, including external config files is GA. Part of #4581. --- CHANGELOG.asciidoc | 2 + filebeat/config/config.go | 3 + filebeat/docs/reload-configuration.asciidoc | 82 +++++++++++-------- metricbeat/docs/reload-configuration.asciidoc | 48 +++++++---- 4 files changed, 83 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index dd9e357c924..e6954fb8530 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -125,6 +125,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di *Filebeat* +- The `filebeat.config_dir` option is deprecated. Use `filebeat.config.prospector` options instead. {pull}5321[5321] + *Heartbeat* *Metricbeat* diff --git a/filebeat/config/config.go b/filebeat/config/config.go index b8dcc1f5ba9..cccbc4c7248 100644 --- a/filebeat/config/config.go +++ b/filebeat/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/elastic/beats/libbeat/cfgfile" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/paths" ) @@ -94,6 +95,8 @@ func (config *Config) FetchConfigs() error { return nil } + cfgwarn.Deprecate("7.0.0", "config_dir is deprecated. Use `filebeat.config.prospectors` instead.") + // If configDir is relative, consider it relative to the config path configDir = paths.Resolve(paths.Config, configDir) diff --git a/filebeat/docs/reload-configuration.asciidoc b/filebeat/docs/reload-configuration.asciidoc index 00242b2cddd..5a3f24ae076 100644 --- a/filebeat/docs/reload-configuration.asciidoc +++ b/filebeat/docs/reload-configuration.asciidoc @@ -1,39 +1,16 @@ [[filebeat-configuration-reloading]] -== Reload the configuration dynamically - -beta[] - -You can configure Filebeat to dynamically reload configuration files when there -are changes. This feature is available for prospector and module configuration -files only. - -To configure this feature, you specify a path -(https://golang.org/pkg/path/filepath/#Glob[Glob]) to watch for configuration -changes. When the files found by the Glob change, new prospectors and/or -modules are started and stopped according to changes in the configuration files. - -This feature is especially useful in container environments where one container -is used to tail logs for services running in other containers on the same host. - -To enable dynamic config reloading, you specify the following options under -`filebeat.config.prospectors` (for prospector configs) or -`filebeat.config.modules` (for module configs): - -`path`:: A Glob that defines the files to check for changes. -`reload.enabled`:: When set to `true`, enables dynamic config reload. -`reload.period`:: Specifies how often the files are checked for changes. Do not -set the `period` to less than 1s because the modification time of files is often -stored in seconds. Setting the `period` to less than 1s will result in -unnecessary overhead. - -See <> and <> for examples. +== Load external configuration files +Filebeat can load external configuration files for prospectors and modules, +which allows you to separate your configuration into multiple smaller +configuration files. See the <> and the +<> sections for details. [float] -[[reload-prospector-config]] +[[load-prospector-config]] === Prospector config -For prospector configurations, you specify reload options in the +For prospector configurations, you specify `path` option in the `filebeat.config.prospectors` section of the +{beatname_lc}.yml+ file. For example: @@ -42,8 +19,6 @@ example: filebeat.config.prospectors: enabled: true path: configs/*.yml - reload.enabled: true - reload.period: 10s ------------------------------------------------------------------------------ Each file found by the Glob must contain a list of one or more prospector @@ -68,10 +43,10 @@ file paths defined. If more than one prospector harvests the same file at the same time, it can lead to unexpected behaviour. [float] -[[reload-module-config]] +[[load-module-config]] === Module config -For module configurations, you specify reload options in the +For module configurations, you specify the `path` option in the `filebeat.config.modules` section of the +{beatname_lc}.yml+ file. For example: [source,yaml] @@ -79,9 +54,8 @@ For module configurations, you specify reload options in the filebeat.config.modules: enabled: true path: prospectors.d/*.yml <1> - reload.enabled: true - reload.period: 10s ------------------------------------------------------------------------------ + <1> If you change the path setting to look for config changes in a different directory, you will not be able to use the <> command to enable and disable module configurations. @@ -99,3 +73,39 @@ definitions. For example: enabled: true var.paths: [/var/log/apache2/error.log*] ------------------------------------------------------------------------------ + +=== Live reloading + +beta[] + +You can configure Filebeat to dynamically reload configuration files when there +are changes. This feature is available for prospector and module configuration +files only. + +To configure this feature, you specify a path +(https://golang.org/pkg/path/filepath/#Glob[Glob]) to watch for configuration +changes. When the files found by the Glob change, new prospectors and/or +modules are started and stopped according to changes in the configuration files. + +This feature is especially useful in container environments where one container +is used to tail logs for services running in other containers on the same host. + +To enable dynamic config reloading, you specify the `path` and `reload` options +under `filebeat.config.prospectors` or `filebeat.config.modules` sections. For +example: + +[source,yaml] +------------------------------------------------------------------------------ +filebeat.config.prospectors: + enabled: true + path: configs/*.yml + reload.enabled: true + reload.period: 10s +------------------------------------------------------------------------------ + +`path`:: A Glob that defines the files to check for changes. +`reload.enabled`:: When set to `true`, enables dynamic config reload. +`reload.period`:: Specifies how often the files are checked for changes. Do not +set the `period` to less than 1s because the modification time of files is often +stored in seconds. Setting the `period` to less than 1s will result in +unnecessary overhead. diff --git a/metricbeat/docs/reload-configuration.asciidoc b/metricbeat/docs/reload-configuration.asciidoc index 2a3e86c1859..ec0410c2249 100644 --- a/metricbeat/docs/reload-configuration.asciidoc +++ b/metricbeat/docs/reload-configuration.asciidoc @@ -1,5 +1,36 @@ [[metricbeat-configuration-reloading]] -== Reload the configuration dynamically +== Load external configuration files + +Metricbeat can load external configuration files for modules, which allows you +to separate your configuration into multiple smaller configuration files. To use +this, you specify the `path` option under `metricbeat.config.modules` in the +main `metricbeat.yml` configuration file. For example: + +[source,yaml] +------------------------------------------------------------------------------ +metricbeat.config.modules: + path: configs/*.yml +------------------------------------------------------------------------------ + +`path`:: A Glob that defines the files to check for changes. + +Each file found by the Glob must contain a list of one or more module +definitions. For example: + +[source,yaml] +------------------------------------------------------------------------------ +- module: system + metricsets: ["cpu"] + enabled: false + perdiod: 1s + +- module: system + metricsets: ["network"] + enabled: true + period: 10s +------------------------------------------------------------------------------ + +=== Live reloading beta[] @@ -34,18 +65,3 @@ set the `period` to less than 1s because the modification time of files is often stored in seconds. Setting the `period` to less than 1s will result in unnecessary overhead. -Each file found by the Glob must contain a list of one or more module -definitions. For example: - -[source,yaml] ------------------------------------------------------------------------------- -- module: system - metricsets: ["cpu"] - enabled: false - perdiod: 1s - -- module: system - metricsets: ["network"] - enabled: true - period: 10s ------------------------------------------------------------------------------- From 9a431f1ab1722450457342a39778216d965e130b Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 5 Oct 2017 10:34:29 +0200 Subject: [PATCH 79/96] Changelog cleanup in master (#5322) This includes the recent releases and eliminates duplicates. --- CHANGELOG.asciidoc | 175 +++++++++++++++++++++++++--------- libbeat/docs/release.asciidoc | 7 ++ 2 files changed, 139 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index e6954fb8530..126d9df0632 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -13,8 +13,6 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di ==== Breaking changes *Affecting all Beats* -- Fix kubernetes matcher registry lookup. {pull}5159[5159] -- Fix data race accessing watched containers. {issue}5147[5147] *Auditbeat* @@ -33,35 +31,20 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di ==== Bugfixes *Affecting all Beats* -- Fix go plugins not loaded when beat starts {pull}4799[4799] -- Eliminate deprecated _default_ mapping in 6.x {pull}4864[4864] -- Register kubernetes `field_format` matcher and remove logger in `Encode` API {pull}4888[4888] -- Add support for `initContainers` in `add_kubernetes_metadata` processor. {issue}4825[4825] -- Fix the `/usr/bin/beatname` script to accept `-d "*"` as a parameter. {issue}5040[5040] -- Combine `fields.yml` properties when they are defined in different sources. {issue}5075[5075] -- Keep Docker & Kubernetes pod metadata after container dies while they are needed by processors. {pull}5084[5084] -- Fix `fields.yml` lookup when using `export template` with a custom `path.config` param. {issue}5089[5089] -- Remove runner creation from every reload check {pull}5141[5141] + +- Fix data race accessing watched containers. {issue}5147[5147] - Remove ID() from Runner interface {issue}5153[5153] *Auditbeat* *Filebeat* + - Fix default paths for redis 4.0.1 logs on macOS {pull}5173[5173] *Heartbeat* *Metricbeat* -- Support `npipe` protocol (Windows) in Docker module. {pull}4751[4751] -- Added missing mongodb configuration file to the `modules.d` folder. {pull}4870[4870] -- Fix wrong MySQL CRUD queries timelion visualization {pull}4857[4857] -- Add new metrics to CPU metricsset {pull}4969[4969] -- Fix a memory allocation issue where more memory was allocated than needed in the windows-perfmon metricset. {issue}5035[5035] -- Don't start metricbeat if external modules config is wrong and reload is disabled {pull}5053[5053] -- Fix kubernetes events module to be able to index time fields properly. {issue}5093[5093] -- The MongoDB module now connects on each fetch, to avoid stopping the whole Metricbeat instance if MongoDB is not up when starting. {pull}5120[5120] -- Fixed `cmd_set` and `cmd_get` being mixed in the Memcache module. {pull}5189[5189] - Change field type of http header from nested to object {pull}5258[5258] *Packetbeat* @@ -76,53 +59,36 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Changed the hashbang used in the beat helper script from `/bin/bash` to `/usr/bin/env bash`. {pull}5051[5051] - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051] - Fix reloader error message to only print on actual error {pull}5066[5066] -- Enable flush timeout by default. {pull}5150[5150] -- Add @metadata.version to events send to Logstash. {pull}5166[5166] -- Add Collectbeat to community beats. {pull}5327[5327] *Auditbeat* -- Changed the number of shards in the default configuration to 3. {issue}5095[5095] - *Filebeat* - Add PostgreSQL module with slowlog support. {pull}4763[4763] - Add Kafka log module. {pull}4885[4885] - Add support for `/var/log/containers/` log path in `add_kubernetes_metadata` processor. {pull}4981[4981] -- Don't start filebeat if external modules/prospectors config is wrong and reload is disabled {pull}5053[5053] - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085] -- Changed the number of shards in the default configuration to 3. {issue}5095[5095] - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085] -- Add `filebeat.registry_flush` setting, to delay the registry updates. {pull}5146[5146] *Heartbeat* -- Changed the number of shards in the default configuration to 1. {issue}5095[5095] - *Metricbeat* - Add graphite protocol metricbeat module. {pull}4734[4734] - Add http server metricset to support push metrics via http. {pull}4770[4770] - Make config object public for graphite and http server {pull}4820[4820] - Add system uptime metricset. {issue}[4848[4848] -- Add `filesystem.ignore_types` to system module for ignoring filesystem types. {issue}4685[4685] - Add experimental `queue` metricset to RabbitMQ module. {pull}4788[4788] - Add additional php-fpm pool status kpis for Metricbeat module {pull}5287[5287] *Packetbeat* -- Changed the number of shards in the default configuration to 3. {issue}5095[5095] - *Winlogbeat* -- Changed the number of shards in the default configuration to 3. {issue}5095[5095] - ==== Deprecated *Affecting all Beats* -- Fix pod name indexer to use both namespace, pod name to frame index key {pull}4775[4775] - *Filebeat* - The `filebeat.config_dir` option is deprecated. Use `filebeat.config.prospector` options instead. {pull}5321[5321] @@ -140,6 +106,61 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di //////////////////////////////////////////////////////////// +[[release-notes-6.0.0-rc1]] +=== Beats version 6.0.0-rc1 +https://github.com/elastic/beats/compare/v6.0.0-beta2...v6.0.0-rc1[View commits] + +==== Bugfixes + +*Affecting all Beats* + +- Fix the `/usr/bin/beatname` script to accept `-d "*"` as a parameter. {issue}5040[5040] +- Combine `fields.yml` properties when they are defined in different sources. {issue}5075[5075] +- Keep Docker & Kubernetes pod metadata after container dies while they are needed by processors. {pull}5084[5084] +- Fix `fields.yml` lookup when using `export template` with a custom `path.config` param. {issue}5089[5089] +- Remove runner creation from every reload check {pull}5141[5141] +- Fix add_kubernetes_metadata matcher registry lookup. {pull}5159[5159] + +*Metricbeat* + +- Fix a memory allocation issue where more memory was allocated than needed in the windows-perfmon metricset. {issue}5035[5035] +- Don't start metricbeat if external modules config is wrong and reload is disabled {pull}5053[5053] +- The MongoDB module now connects on each fetch, to avoid stopping the whole Metricbeat instance if MongoDB is not up when starting. {pull}5120[5120] +- Fix kubernetes events module to be able to index time fields properly. {issue}5093[5093] +- Fixed `cmd_set` and `cmd_get` being mixed in the Memcache module. {pull}5189[5189] + + +==== Added + +*Affecting all Beats* + +- Enable flush timeout by default. {pull}5150[5150] +- Add @metadata.version to events send to Logstash. {pull}5166[5166] + +*Auditbeat* + +- Changed the number of shards in the default configuration to 3. {issue}5095[5095] +- Add support for receiving audit events using a multicast socket. {issue}4850[4850] + +*Filebeat* + +- Changed the number of shards in the default configuration to 3. {issue}5095[5095] +- Don't start filebeat if external modules/prospectors config is wrong and reload is disabled {pull}5053[5053] +- Add `filebeat.registry_flush` setting, to delay the registry updates. {pull}5146[5146] + +*Heartbeat* + +- Changed the number of shards in the default configuration to 1. {issue}5095[5095] + +*Packetbeat* + +- Changed the number of shards in the default configuration to 3. {issue}5095[5095] + +*Winlogbeat* + +- Changed the number of shards in the default configuration to 3. {issue}5095[5095] + + [[release-notes-6.0.0-beta2]] === Beats version 6.0.0-beta2 https://github.com/elastic/beats/compare/v6.0.0-beta1...v6.0.0-beta2[View commits] @@ -164,6 +185,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...v6.0.0-beta2[View commit - Fix go plugins not loaded when beat starts {pull}4799[4799] - Add support for `initContainers` in `add_kubernetes_metadata` processor. {issue}4825[4825] - Eliminate deprecated _default_ mapping in 6.x {pull}4864[4864] +- Fix pod name indexer to use both namespace, pod name to frame index key {pull}4775[4775] *Heartbeat* @@ -197,12 +219,6 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...v6.0.0-beta2[View commit - Add `filesystem.ignore_types` to system module for ignoring filesystem types. {issue}4685[4685] - Add support to exclude labels from kubernetes pod metadata. {pull}4757[4757] -==== Deprecated - -*Affecting all Beats* - -- Fix pod name indexer to use both namespace, pod name to frame index key {pull}4775[4775] - [[release-notes-6.0.0-beta1]] === Beats version 6.0.0-beta1 @@ -530,6 +546,79 @@ https://github.com/elastic/beats/compare/v5.4.0...v6.0.0-alpha1[View commits] - Prospector reloading only works properly with new files. {pull}3546[3546] +[[release-notes-5.6.2]] +=== Beats version 5.6.2 +https://github.com/elastic/beats/compare/v5.6.1...v5.6.2[View commits] + +No changes in this release. + +[[release-notes-5.6.1]] +=== Beats version 5.6.1 +https://github.com/elastic/beats/compare/v5.6.0...v5.6.1[View commits] + +No changes in this release. + +[[release-notes-5.6.0]] +=== Beats version 5.6.0 +https://github.com/elastic/beats/compare/v5.5.3...v5.6.0[View commits] + +==== Breaking changes + +*Affecting all Beats* + +- The _all.norms setting in the Elasticsearch template is no longer disabled. + This increases the storage size with one byte per document, but allows for a + better upgrade experience to 6.0. {issue}4901[4901] + + +==== Bugfixes + +*Filebeat* + +- Fix issue where the `fileset.module` could have the wrong value. {issue}4761[4761] + +*Packetbeat* + +- Update flow timestamp on each packet being received. {issue}4895[4895] + +*Metricbeat* + +- Fix a debug statement that said a module wrapper had stopped when it hadn't. {pull}4264[4264] +- Use MemAvailable value from /proc/meminfo on Linux 3.14. {pull}4316[4316] +- Fix panic when events were dropped by filters. {issue}4327[4327] + +==== Added + +*Affecting all Beats* + +- Add option to the import_dashboards script to load the dashboards via Kibana API. {pull}4682[4682] + +*Filebeat* + +- Add support for loading Xpack Machine Learning configurations from the modules, and added sample configurations for the Nginx module. {pull}4506[4506] {pull}4609[4609] +- Add ability to parse nginx logs exposing the X-Forwarded-For header instead of the remote address. {pull}4351[4351] + +*Metricbeat* + +- Add `filesystem.ignore_types` to system module for ignoring filesystem types. {issue}4685[4685] + +==== Deprecated + +*Affecting all Beats* + +- Loading more than one output is deprecated and will be removed in 6.0. {pull}4907[4907] + +[[release-notes-5.5.3]] +=== Beats version 5.5.3 +https://github.com/elastic/beats/compare/v5.5.2...v5.5.3[View commits] + +No changes in this release. + +[[release-notes-5.5.2]] +=== Beats version 5.5.2 +https://github.com/elastic/beats/compare/v5.5.1...v5.5.2[View commits] + +No changes in this release. [[release-notes-5.5.1]] === Beats version 5.5.1 https://github.com/elastic/beats/compare/v5.5.0...v5.5.1[View commits] diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index 3f52e596aaf..950b746aacf 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -6,9 +6,16 @@ -- This section summarizes the changes in each release. +* <> +* <> * <> * <> * <> +* <> +* <> +* <> +* <> +* <> * <> * <> * <> From ce9e47b83ab5a7dd09c99832cdcf0ebdbc485cb0 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 5 Oct 2017 11:39:13 +0200 Subject: [PATCH 80/96] Do not require template if index change and template disabled (#5319) * Do not require template if index change and template disabled If the elasticsearch index name is changed, its required to set a template. The error message was also shown when template itself are disabled. This is now changed that if templates are disabled, no error is returned. Closes https://github.com/elastic/beats/issues/5308 * Add Changelog --- CHANGELOG.asciidoc | 1 + libbeat/cmd/instance/beat.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 126d9df0632..becbbb5e4f9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -34,6 +34,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Fix data race accessing watched containers. {issue}5147[5147] - Remove ID() from Runner interface {issue}5153[5153] +- Do not require template if index change and template disabled {pull}5319[5319] *Auditbeat* diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 731f9330109..62671b01c62 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -575,7 +575,7 @@ func (b *Beat) registerTemplateLoading() error { return err } - if esCfg.Index != "" && (cfg.Name == "" || cfg.Pattern == "") { + if esCfg.Index != "" && (cfg.Name == "" || cfg.Pattern == "") && (b.Config.Template == nil || b.Config.Template.Enabled()) { return fmt.Errorf("setup.template.name and setup.template.pattern have to be set if index name is modified.") } From 4778c51fa81f47ec519c9cf538cc19b4d391d697 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Fri, 6 Oct 2017 14:24:10 +0200 Subject: [PATCH 81/96] Improvements and fixes to the Host Overview dashboard (#5340) * Use `beat.name` instead of `beat.hostname` in visualizations. See #5276 for the motivation * Add a "tip" widget that tells user how they can select another host. * Automatically set a search pattern of the form `beat.name: "HOSTNAME"` where HOSTNAME is the Beat name (hostname) that uploads the dashboards. The way the last point works is that I saved the dashbaord using this filter: `beat.name:"CHANGEME_HOSTNAME"`. The kibana loader code does a string replacement and replaces `CHANGEME_HOSTNAME` with the actual hostname. The disadvantage of this approach (vs doing JSON parsing) is that we must remember to always save that dashboard with the `CHANGEME_HOSTNAME` wildcard in place. I have added a unit test to the system module that checks for that, so it should be relatively hard for us to forget. Closes #5276. --- CHANGELOG.asciidoc | 2 + libbeat/cmd/instance/beat.go | 2 +- libbeat/dashboards/dashboards.go | 4 +- libbeat/dashboards/kibana_loader.go | 17 +++-- libbeat/dashboards/modify_json.go | 14 ++++ libbeat/dashboards/modify_json_test.go | 54 ++++++++++++++ .../Metricbeat-containers-overview.json | 12 +-- .../dashboard/Metricbeat-host-overview.json | 74 ++++++++++--------- .../dashboard/Metricbeat-system-overview.json | 32 ++++---- metricbeat/module/system/util_test.go | 16 ++++ 10 files changed, 164 insertions(+), 63 deletions(-) create mode 100644 libbeat/dashboards/modify_json_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index becbbb5e4f9..ef534fa6fc6 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -47,6 +47,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di *Metricbeat* - Change field type of http header from nested to object {pull}5258[5258] +- Use `beat.name` instead of `beat.hostname` in the Host Overview dashboard. {pull}5340[5340] *Packetbeat* @@ -81,6 +82,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add system uptime metricset. {issue}[4848[4848] - Add experimental `queue` metricset to RabbitMQ module. {pull}4788[4788] - Add additional php-fpm pool status kpis for Metricbeat module {pull}5287[5287] +- Auto-select a hostname (based on the host on which the Beat is running) in the Host Overview dashboard. {pull}5340[5340] *Packetbeat* diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 62671b01c62..d02a140aaf4 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -537,7 +537,7 @@ func (b *Beat) loadDashboards(force bool) error { if b.Config.Output.Name() == "elasticsearch" { esConfig = b.Config.Output.Config() } - err := dashboards.ImportDashboards(b.Info.Beat, b.Info.Version, paths.Resolve(paths.Home, ""), + err := dashboards.ImportDashboards(b.Info.Beat, b.Info.Name, paths.Resolve(paths.Home, ""), b.Config.Kibana, esConfig, b.Config.Dashboards, nil) if err != nil { return fmt.Errorf("Error importing Kibana dashboards: %v", err) diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index 080885deae1..a867212f631 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -10,7 +10,7 @@ import ( "github.com/elastic/beats/libbeat/logp" ) -func ImportDashboards(beatName, beatVersion, homePath string, +func ImportDashboards(beatName, hostname, homePath string, kibanaConfig *common.Config, esConfig *common.Config, dashboardsConfig *common.Config, msgOutputter MessageOutputter) error { @@ -62,7 +62,7 @@ func ImportDashboards(beatName, beatVersion, homePath string, kibanaConfig.SetString("password", -1, esLoader.client.Password) } - kibanaLoader, err := NewKibanaLoader(kibanaConfig, &dashConfig, msgOutputter) + kibanaLoader, err := NewKibanaLoader(kibanaConfig, &dashConfig, hostname, msgOutputter) if err != nil { return fmt.Errorf("fail to create the Kibana loader: %v", err) } diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index ba58a931b7e..065a0a8bbd8 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -17,10 +17,11 @@ type KibanaLoader struct { client *kibana.Client config *Config version string + hostname string msgOutputter MessageOutputter } -func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, msgOutputter MessageOutputter) (*KibanaLoader, error) { +func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) { if cfg == nil || !cfg.Enabled() { return nil, fmt.Errorf("Kibana is not configured or enabled") @@ -35,6 +36,7 @@ func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, msgOutputter client: client, config: dashboardsConfig, version: client.GetVersion(), + hostname: hostname, msgOutputter: msgOutputter, } @@ -50,13 +52,13 @@ func (loader KibanaLoader) ImportIndex(file string) error { // read json file reader, err := ioutil.ReadFile(file) if err != nil { - return fmt.Errorf("fail to read index-pattern: %v", err) + return fmt.Errorf("fail to read index-pattern from file %s: %v", file, err) } var indexContent common.MapStr err = json.Unmarshal(reader, &indexContent) if err != nil { - return fmt.Errorf("fail to unmarshal the index content: %v", err) + return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err) } indexContent = ReplaceIndexInIndexPattern(loader.config.Index, indexContent) @@ -72,16 +74,21 @@ func (loader KibanaLoader) ImportDashboard(file string) error { // read json file reader, err := ioutil.ReadFile(file) if err != nil { - return fmt.Errorf("fail to read index-pattern: %v", err) + return fmt.Errorf("fail to read dashboard from file %s: %v", file, err) } var content common.MapStr err = json.Unmarshal(reader, &content) if err != nil { - return fmt.Errorf("fail to unmarshal the index content: %v", err) + return fmt.Errorf("fail to unmarshal the dashboard content from file %s: %v", file, err) } content = ReplaceIndexInDashboardObject(loader.config.Index, content) + content, err = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content) + if err != nil { + return fmt.Errorf("fail to replace the hostname in dashboard %s: %v", file, err) + } + return loader.client.ImportJSON(importAPI, params, content) } diff --git a/libbeat/dashboards/modify_json.go b/libbeat/dashboards/modify_json.go index 2a219bfdfeb..75c9ff15655 100644 --- a/libbeat/dashboards/modify_json.go +++ b/libbeat/dashboards/modify_json.go @@ -1,6 +1,7 @@ package dashboards import ( + "bytes" "encoding/json" "fmt" @@ -103,3 +104,16 @@ func ReplaceIndexInDashboardObject(index string, content common.MapStr) common.M } return content } + +func ReplaceStringInDashboard(old, new string, content common.MapStr) (common.MapStr, error) { + marshaled, err := json.Marshal(content) + if err != nil { + return nil, fmt.Errorf("fail to marshal dashboard object: %v", content) + } + + replaced := bytes.Replace(marshaled, []byte(old), []byte(new), -1) + + var result common.MapStr + err = json.Unmarshal(replaced, &result) + return result, nil +} diff --git a/libbeat/dashboards/modify_json_test.go b/libbeat/dashboards/modify_json_test.go new file mode 100644 index 00000000000..9e8c3dcf610 --- /dev/null +++ b/libbeat/dashboards/modify_json_test.go @@ -0,0 +1,54 @@ +package dashboards + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +func TestReplaceStringInDashboard(t *testing.T) { + tests := []struct { + content common.MapStr + old string + new string + expected common.MapStr + }{ + { + content: common.MapStr{"test": "CHANGEME"}, + old: "CHANGEME", + new: "hostname", + expected: common.MapStr{"test": "hostname"}, + }, + { + content: common.MapStr{"test": "hello"}, + old: "CHANGEME", + new: "hostname", + expected: common.MapStr{"test": "hello"}, + }, + { + content: common.MapStr{"test": map[string]interface{}{"key": "\"CHANGEME\""}}, + old: "CHANGEME", + new: "hostname.local", + expected: common.MapStr{"test": map[string]interface{}{"key": "\"hostname.local\""}}, + }, + { + content: common.MapStr{ + "kibanaSavedObjectMeta": map[string]interface{}{ + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"beat.name:\\\"CHANGEME_HOSTNAME\\\"\",\"language\":\"lucene\"}}"}}, + + old: "CHANGEME_HOSTNAME", + new: "hostname.local", + expected: common.MapStr{ + "kibanaSavedObjectMeta": map[string]interface{}{ + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"beat.name:\\\"hostname.local\\\"\",\"language\":\"lucene\"}}"}}, + }, + } + + for _, test := range tests { + result, err := ReplaceStringInDashboard(test.old, test.new, test.content) + assert.NoError(t, err) + assert.Equal(t, test.expected, result) + } +} diff --git a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-containers-overview.json b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-containers-overview.json index 4366bb67ef2..9d5fd650d14 100644 --- a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-containers-overview.json +++ b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-containers-overview.json @@ -13,7 +13,7 @@ }, "id": "Container-CPU-usage", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -28,7 +28,7 @@ }, "id": "System-Navigation", "type": "visualization", - "version": 6 + "version": 3 }, { "attributes": { @@ -43,7 +43,7 @@ }, "id": "Container-Memory-stats", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -58,7 +58,7 @@ }, "id": "Container-Block-IO", "type": "visualization", - "version": 3 + "version": 1 }, { "attributes": { @@ -76,8 +76,8 @@ }, "id": "CPU-slash-Memory-per-container", "type": "dashboard", - "version": 4 + "version": 1 } ], - "version": "6.0.0-beta1-SNAPSHOT" + "version": "6.0.0-rc1-SNAPSHOT" } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-host-overview.json b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-host-overview.json index c3fae5ecd76..c1b4558c772 100644 --- a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-host-overview.json +++ b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-host-overview.json @@ -13,7 +13,7 @@ }, "id": "6b7b9a40-faa1-11e6-86b1-cd7735ff7e23", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -28,7 +28,7 @@ }, "id": "4d546850-1b15-11e7-b09e-037021c4f8df", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -43,7 +43,7 @@ }, "id": "089b85d0-1b16-11e7-b09e-037021c4f8df", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -58,7 +58,7 @@ }, "id": "bfa5e400-1b16-11e7-b09e-037021c4f8df", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -73,7 +73,7 @@ }, "id": "e0f001c0-1b18-11e7-b09e-037021c4f8df", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -88,7 +88,7 @@ }, "id": "2e224660-1b19-11e7-b09e-037021c4f8df", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -103,7 +103,7 @@ }, "id": "ab2d1e90-1b1a-11e7-b09e-037021c4f8df", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -118,7 +118,7 @@ }, "id": "4e4bb1e0-1b1b-11e7-b09e-037021c4f8df", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -133,7 +133,7 @@ }, "id": "26732e20-1b91-11e7-bec4-a5e9ec5cab8b", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -148,7 +148,7 @@ }, "id": "83e12df0-1b91-11e7-bec4-a5e9ec5cab8b", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -163,7 +163,7 @@ }, "id": "d3166e80-1b91-11e7-bec4-a5e9ec5cab8b", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -178,7 +178,7 @@ }, "id": "522ee670-1b92-11e7-bec4-a5e9ec5cab8b", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -193,7 +193,7 @@ }, "id": "1aae9140-1b93-11e7-8ada-3df93aab833e", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -208,7 +208,7 @@ }, "id": "34f97ee0-1b96-11e7-8ada-3df93aab833e", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -223,7 +223,7 @@ }, "id": "System-Navigation", "type": "visualization", - "version": 6 + "version": 3 }, { "attributes": { @@ -238,7 +238,7 @@ }, "id": "19e123b0-4d5a-11e7-aee5-fdc812cc3bec", "type": "visualization", - "version": 3 + "version": 1 }, { "attributes": { @@ -268,7 +268,7 @@ }, "id": "825fdb80-4d1d-11e7-b5f2-2b7c1895bf32", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -283,7 +283,7 @@ }, "id": "96976150-4d5d-11e7-aa29-87a97a796de6", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -298,7 +298,7 @@ }, "id": "99381c80-4d60-11e7-9a4c-ed99bbcaa42b", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -313,7 +313,7 @@ }, "id": "c5e3cf90-4d60-11e7-9a4c-ed99bbcaa42b", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -328,6 +328,21 @@ }, "id": "590a60f0-5d87-11e7-8884-1bb4c3b890e4", "type": "visualization", + "version": 1 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + }, + "title": "Tip [Metricbeat System]", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"title\":\"Tip [Metricbeat System]\",\"type\":\"markdown\",\"params\":{\"fontSize\":12,\"markdown\":\"**TIP:** To select another host, go to the [System Overview](#/dashboard/Metricbeat-system-overview) dashboard and double-click a host name.\"},\"aggs\":[]}" + }, + "id": "3d65d450-a9c3-11e7-af20-67db8aecb295", + "type": "visualization", "version": 2 }, { @@ -335,26 +350,19 @@ "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"\",\"default_field\":\"*\"}},\"language\":\"lucene\"}}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"beat.name:\\\"CHANGEME_HOSTNAME\\\"\"}}" }, "optionsJSON": "{\"darkTheme\":false}", - "panelsJSON": "[{\"col\":1,\"id\":\"6b7b9a40-faa1-11e6-86b1-cd7735ff7e23\",\"panelIndex\":1,\"row\":12,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"4d546850-1b15-11e7-b09e-037021c4f8df\",\"panelIndex\":2,\"row\":6,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"089b85d0-1b16-11e7-b09e-037021c4f8df\",\"panelIndex\":3,\"row\":12,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"bfa5e400-1b16-11e7-b09e-037021c4f8df\",\"panelIndex\":4,\"row\":9,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"e0f001c0-1b18-11e7-b09e-037021c4f8df\",\"panelIndex\":5,\"row\":15,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2e224660-1b19-11e7-b09e-037021c4f8df\",\"panelIndex\":6,\"row\":15,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ab2d1e90-1b1a-11e7-b09e-037021c4f8df\",\"panelIndex\":7,\"row\":6,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"4e4bb1e0-1b1b-11e7-b09e-037021c4f8df\",\"panelIndex\":8,\"row\":9,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"26732e20-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":9,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"83e12df0-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":10,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":3,\"id\":\"d3166e80-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":11,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"522ee670-1b92-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":12,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"1aae9140-1b93-11e7-8ada-3df93aab833e\",\"panelIndex\":13,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"34f97ee0-1b96-11e7-8ada-3df93aab833e\",\"panelIndex\":14,\"row\":4,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":16,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"19e123b0-4d5a-11e7-aee5-fdc812cc3bec\",\"panelIndex\":21,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":3,\"id\":\"d2e80340-4d5c-11e7-aa29-87a97a796de6\",\"panelIndex\":22,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"825fdb80-4d1d-11e7-b5f2-2b7c1895bf32\",\"panelIndex\":23,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":11,\"id\":\"96976150-4d5d-11e7-aa29-87a97a796de6\",\"panelIndex\":25,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"99381c80-4d60-11e7-9a4c-ed99bbcaa42b\",\"panelIndex\":27,\"row\":18,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"c5e3cf90-4d60-11e7-9a4c-ed99bbcaa42b\",\"panelIndex\":28,\"row\":18,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"590a60f0-5d87-11e7-8884-1bb4c3b890e4\",\"panelIndex\":29,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"}]", - "refreshInterval": { - "display": "Off", - "pause": false, - "value": 0 - }, - "timeFrom": "now-15m", - "timeRestore": true, - "timeTo": "now", + "panelsJSON": "[{\"col\":1,\"id\":\"6b7b9a40-faa1-11e6-86b1-cd7735ff7e23\",\"panelIndex\":1,\"row\":12,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"4d546850-1b15-11e7-b09e-037021c4f8df\",\"panelIndex\":2,\"row\":6,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"089b85d0-1b16-11e7-b09e-037021c4f8df\",\"panelIndex\":3,\"row\":12,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"bfa5e400-1b16-11e7-b09e-037021c4f8df\",\"panelIndex\":4,\"row\":9,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"e0f001c0-1b18-11e7-b09e-037021c4f8df\",\"panelIndex\":5,\"row\":15,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2e224660-1b19-11e7-b09e-037021c4f8df\",\"panelIndex\":6,\"row\":15,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ab2d1e90-1b1a-11e7-b09e-037021c4f8df\",\"panelIndex\":7,\"row\":6,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"4e4bb1e0-1b1b-11e7-b09e-037021c4f8df\",\"panelIndex\":8,\"row\":9,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"26732e20-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":9,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"83e12df0-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":10,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":3,\"id\":\"d3166e80-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":11,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"522ee670-1b92-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":12,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"1aae9140-1b93-11e7-8ada-3df93aab833e\",\"panelIndex\":13,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"34f97ee0-1b96-11e7-8ada-3df93aab833e\",\"panelIndex\":14,\"row\":4,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":16,\"row\":1,\"size_x\":6,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"19e123b0-4d5a-11e7-aee5-fdc812cc3bec\",\"panelIndex\":21,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":3,\"id\":\"d2e80340-4d5c-11e7-aa29-87a97a796de6\",\"panelIndex\":22,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"825fdb80-4d1d-11e7-b5f2-2b7c1895bf32\",\"panelIndex\":23,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":11,\"id\":\"96976150-4d5d-11e7-aa29-87a97a796de6\",\"panelIndex\":25,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"99381c80-4d60-11e7-9a4c-ed99bbcaa42b\",\"panelIndex\":27,\"row\":18,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"c5e3cf90-4d60-11e7-9a4c-ed99bbcaa42b\",\"panelIndex\":28,\"row\":18,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"590a60f0-5d87-11e7-8884-1bb4c3b890e4\",\"panelIndex\":29,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"3d65d450-a9c3-11e7-af20-67db8aecb295\",\"panelIndex\":30,\"row\":1,\"size_x\":6,\"size_y\":1,\"type\":\"visualization\"}]", + "timeRestore": false, "title": "[Metricbeat System] Host overview", "uiStateJSON": "{\"P-29\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}}", "version": 1 }, "id": "79ffd6e0-faa0-11e6-947f-177f697178b8", "type": "dashboard", - "version": 3 + "version": 12 } ], - "version": "6.0.0-beta1-SNAPSHOT" -} + "version": "6.0.0-rc1-SNAPSHOT" +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-system-overview.json b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-system-overview.json index e1f7122bf5a..b9688a8c1b7 100644 --- a/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-system-overview.json +++ b/metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-system-overview.json @@ -13,7 +13,7 @@ }, "id": "System-Navigation", "type": "visualization", - "version": 6 + "version": 3 }, { "attributes": { @@ -24,7 +24,7 @@ "title": "Number of hosts [Metricbeat System]", "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", "version": 1, - "visState": "{\"title\":\"Number of hosts [Metricbeat System]\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"63\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"beat.hostname\",\"customLabel\":\"Number of hosts\"}}]}" + "visState": "{\"title\":\"Number of hosts [Metricbeat System]\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"63\",\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"beat.name\",\"customLabel\":\"Number of hosts\"}}]}" }, "id": "c6f2ffd0-4d17-11e7-a196-69b9a7a020a9", "type": "visualization", @@ -39,7 +39,7 @@ "title": "Top Hosts By Memory (Realtime) [Metricbeat System]", "uiStateJSON": "{}", "version": 1, - "visState": "{\"title\":\"Top Hosts By Memory (Realtime) [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"id\":\"31e5afa0-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"top_n\",\"series\":[{\"id\":\"31e5afa1-1b1c-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.memory.actual.used.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"beat.hostname\",\"terms_order_by\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"terms_size\":\"10\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"value\":0,\"id\":\"33349dd0-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"},{\"value\":0.6,\"id\":\"997dc440-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(254,146,0,1)\",\"opperator\":\"gte\"},{\"value\":0.85,\"id\":\"a10d7f20-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(211,49,21,1)\",\"opperator\":\"gte\"}],\"drilldown_url\":\"../app/kibana#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8?_a=(query:(query_string:(analyze_wildcard:!t,query:'beat.hostname:\\\"{{key}}\\\"')))\",\"filter\":\"\",\"show_grid\":1},\"aggs\":[]}" + "visState": "{\"title\":\"Top Hosts By Memory (Realtime) [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"id\":\"31e5afa0-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"top_n\",\"series\":[{\"id\":\"31e5afa1-1b1c-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.memory.actual.used.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"beat.name\",\"terms_order_by\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"terms_size\":\"10\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"value\":0,\"id\":\"33349dd0-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"},{\"value\":0.6,\"id\":\"997dc440-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(254,146,0,1)\",\"opperator\":\"gte\"},{\"value\":0.85,\"id\":\"a10d7f20-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(211,49,21,1)\",\"opperator\":\"gte\"}],\"drilldown_url\":\"../app/kibana#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8?_a=(query:(query_string:(analyze_wildcard:!t,query:'beat.name:\\\"{{key}}\\\"')))\",\"filter\":\"\",\"show_grid\":1},\"aggs\":[]}" }, "id": "fe064790-1b1f-11e7-bec4-a5e9ec5cab8b", "type": "visualization", @@ -49,12 +49,12 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\n \"query\": {\n \"query_string\": {\n \"query\": \"*\"\n }\n },\n \"filter\": []\n}" + "searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" }, "title": "Top Hosts By CPU (Realtime) [Metricbeat System]", "uiStateJSON": "{}", "version": 1, - "visState": "{\n \"title\": \"Top Hosts By CPU (Realtime)\",\n \"type\": \"metrics\",\n \"params\": {\n \"id\": \"31e5afa0-1b1c-11e7-b09e-037021c4f8df\",\n \"type\": \"top_n\",\n \"series\": [\n {\n \"id\": \"31e5afa1-1b1c-11e7-b09e-037021c4f8df\",\n \"color\": \"#68BC00\",\n \"split_mode\": \"terms\",\n \"metrics\": [\n {\n \"id\": \"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\n \"type\": \"avg\",\n \"field\": \"system.cpu.user.pct\"\n }\n ],\n \"seperate_axis\": 0,\n \"axis_position\": \"right\",\n \"formatter\": \"percent\",\n \"chart_type\": \"line\",\n \"line_width\": 1,\n \"point_size\": 1,\n \"fill\": 0.5,\n \"stacked\": \"none\",\n \"terms_field\": \"beat.hostname\",\n \"terms_order_by\": \"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\n \"terms_size\": \"10\"\n }\n ],\n \"time_field\": \"@timestamp\",\n \"index_pattern\": \"*\",\n \"interval\": \"auto\",\n \"axis_position\": \"left\",\n \"axis_formatter\": \"number\",\n \"show_legend\": 1,\n \"bar_color_rules\": [\n {\n \"value\": 0,\n \"id\": \"33349dd0-1b1c-11e7-b09e-037021c4f8df\",\n \"bar_color\": \"rgba(104,188,0,1)\",\n \"opperator\": \"gte\"\n },\n {\n \"value\": 0.6,\n \"id\": \"997dc440-1b1c-11e7-b09e-037021c4f8df\",\n \"bar_color\": \"rgba(254,146,0,1)\",\n \"opperator\": \"gte\"\n },\n {\n \"value\": 0.85,\n \"id\": \"a10d7f20-1b1c-11e7-b09e-037021c4f8df\",\n \"bar_color\": \"rgba(211,49,21,1)\",\n \"opperator\": \"gte\"\n }\n ],\n \"drilldown_url\": \"../app/kibana#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8?_a=(query:(query_string:(analyze_wildcard:!t,query:'beat.hostname:\\\"{{key}}\\\"')))\",\n \"filter\": \"\"\n },\n \"aggs\": [],\n \"listeners\": {}\n}" + "visState": "{\"title\":\"Top Hosts By CPU (Realtime) [Metricbeat System]\",\"type\":\"metrics\",\"params\":{\"id\":\"31e5afa0-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"top_n\",\"series\":[{\"id\":\"31e5afa1-1b1c-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.cpu.user.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"beat.name\",\"terms_order_by\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"terms_size\":\"10\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"value\":0,\"id\":\"33349dd0-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"},{\"value\":0.6,\"id\":\"997dc440-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(254,146,0,1)\",\"opperator\":\"gte\"},{\"value\":0.85,\"id\":\"a10d7f20-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(211,49,21,1)\",\"opperator\":\"gte\"}],\"drilldown_url\":\"../app/kibana#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8?_a=(query:(query_string:(analyze_wildcard:!t,query:'beat.name:\\\"{{key}}\\\"')))\",\"filter\":\"\",\"show_grid\":1},\"aggs\":[]}" }, "id": "855899e0-1b1c-11e7-b09e-037021c4f8df", "type": "visualization", @@ -69,11 +69,11 @@ "title": "Hosts histogram by CPU usage [Metricbeat System]", "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0% - 5%\":\"rgb(247,252,245)\",\"5% - 10%\":\"rgb(199,233,192)\",\"10% - 15%\":\"rgb(116,196,118)\",\"15% - 20%\":\"rgb(35,139,69)\"}}}", "version": 1, - "visState": "{\"title\":\"Hosts histogram by CPU usage [Metricbeat System]\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Greens\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"#555\"}}],\"type\":\"heatmap\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.cpu.user.pct\",\"customLabel\":\"CPU usage\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"beat.hostname\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Hosts\"}}]}" + "visState": "{\"title\":\"Hosts histogram by CPU usage [Metricbeat System]\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Greens\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"#555\"}}],\"type\":\"heatmap\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.cpu.user.pct\",\"customLabel\":\"CPU usage\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"beat.name\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Hosts\"}}]}" }, "id": "7cdb1330-4d1a-11e7-a196-69b9a7a020a9", "type": "visualization", - "version": 2 + "version": 1 }, { "attributes": { @@ -88,7 +88,7 @@ }, "id": "522ee670-1b92-11e7-bec4-a5e9ec5cab8b", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -103,7 +103,7 @@ }, "id": "1aae9140-1b93-11e7-8ada-3df93aab833e", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -118,7 +118,7 @@ }, "id": "825fdb80-4d1d-11e7-b5f2-2b7c1895bf32", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -133,7 +133,7 @@ }, "id": "d3166e80-1b91-11e7-bec4-a5e9ec5cab8b", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { @@ -148,26 +148,26 @@ }, "id": "83e12df0-1b91-11e7-bec4-a5e9ec5cab8b", "type": "visualization", - "version": 3 + "version": 2 }, { "attributes": { "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"}}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}}}" }, "optionsJSON": "{\"darkTheme\":false}", "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":9,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"c6f2ffd0-4d17-11e7-a196-69b9a7a020a9\",\"panelIndex\":11,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"fe064790-1b1f-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":12,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"855899e0-1b1c-11e7-b09e-037021c4f8df\",\"panelIndex\":13,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"7cdb1330-4d1a-11e7-a196-69b9a7a020a9\",\"panelIndex\":14,\"row\":9,\"size_x\":12,\"size_y\":6,\"type\":\"visualization\"},{\"col\":9,\"id\":\"522ee670-1b92-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":16,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":11,\"id\":\"1aae9140-1b93-11e7-8ada-3df93aab833e\",\"panelIndex\":17,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"825fdb80-4d1d-11e7-b5f2-2b7c1895bf32\",\"panelIndex\":18,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"d3166e80-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":19,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":3,\"id\":\"83e12df0-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":20,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"}]", "timeRestore": false, "title": "[Metricbeat System] Overview", - "uiStateJSON": "{\"P-11\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-12\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-14\":{\"vis\":{\"defaultColors\":{\"0% - 8.75%\":\"rgb(247,252,245)\",\"17.5% - 26.25%\":\"rgb(116,196,118)\",\"26.25% - 35%\":\"rgb(35,139,69)\",\"8.75% - 17.5%\":\"rgb(199,233,192)\"}}},\"P-16\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-2\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-3\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}}", + "uiStateJSON": "{\"P-11\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-12\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-14\":{\"vis\":{\"defaultColors\":{\"0% - 15%\":\"rgb(247,252,245)\",\"15% - 30%\":\"rgb(199,233,192)\",\"30% - 45%\":\"rgb(116,196,118)\",\"45% - 60%\":\"rgb(35,139,69)\"}}},\"P-16\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-2\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-3\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}}", "version": 1 }, "id": "Metricbeat-system-overview", "type": "dashboard", - "version": 3 + "version": 2 } ], - "version": "6.0.0-beta1-SNAPSHOT" + "version": "6.0.0-rc1-SNAPSHOT" } diff --git a/metricbeat/module/system/util_test.go b/metricbeat/module/system/util_test.go index 41cf5ff86f3..7d73e36918e 100644 --- a/metricbeat/module/system/util_test.go +++ b/metricbeat/module/system/util_test.go @@ -4,6 +4,8 @@ package system import ( + "bytes" + "io/ioutil" "runtime" "testing" @@ -131,3 +133,17 @@ func TestRound(t *testing.T) { assert.EqualValues(t, 1234.5, Round(1234.50004)) assert.EqualValues(t, 1234.5001, Round(1234.50005)) } + +// Checks that the Host Overview dashboard contains the CHANGEME_HOSTNAME variable +// that the dashboard loader code magically changes to the hostname on which the Beat +// is running. +func TestHostDashboardHasChangeableHost(t *testing.T) { + dashPath := "_meta/kibana/default/dashboard/Metricbeat-host-overview.json" + contents, err := ioutil.ReadFile(dashPath) + if err != nil { + t.Fatalf("Error reading file %s: %v", dashPath, err) + } + if !bytes.Contains(contents, []byte("CHANGEME_HOSTNAME")) { + t.Errorf("Dashboard '%s' doesn't contain string 'CHANGEME_HOSTNAME'. See elastic/beats#5340", dashPath) + } +} From 28cee61beaf737380b319d8ecd6ce3520b8b3306 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 Oct 2017 06:13:28 -0400 Subject: [PATCH 82/96] Add support TLS renegotiation (#5353) This PR adds support for enabling TLS renegotiation. The setting is `ssl.renegotiation` and the options are `never` (default), `once`, and `freely`. This exposes the three options from https://golang.org/pkg/crypto/tls/#RenegotiationSupport. Fixes #4386 --- CHANGELOG.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 15 +++++++++++++++ filebeat/filebeat.reference.yml | 15 +++++++++++++++ heartbeat/heartbeat.reference.yml | 15 +++++++++++++++ libbeat/_meta/config.reference.yml | 15 +++++++++++++++ libbeat/docs/shared-ssl-config.asciidoc | 9 +++++++++ libbeat/outputs/tls.go | 20 ++++++++++++++++++++ libbeat/outputs/tls_test.go | 8 ++++++++ libbeat/outputs/transport/tls.go | 4 ++++ metricbeat/metricbeat.reference.yml | 15 +++++++++++++++ packetbeat/packetbeat.reference.yml | 15 +++++++++++++++ winlogbeat/winlogbeat.reference.yml | 15 +++++++++++++++ 12 files changed, 147 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index ef534fa6fc6..5934e2f3dfe 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -61,6 +61,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Changed the hashbang used in the beat helper script from `/bin/bash` to `/usr/bin/env bash`. {pull}5051[5051] - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051] - Fix reloader error message to only print on actual error {pull}5066[5066] +- Add support for enabling TLS renegotiation. {issue}4386[4386] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 0c4d78055ce..42920c139c4 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -281,6 +281,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #----------------------------- Logstash output --------------------------------- #output.logstash: @@ -357,6 +361,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. @@ -489,6 +497,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. @@ -586,6 +598,9 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index d1d93174ae1..dee3140514f 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -701,6 +701,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #----------------------------- Logstash output --------------------------------- #output.logstash: @@ -777,6 +781,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. @@ -909,6 +917,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. @@ -1006,6 +1018,9 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 5e1f7d1e02a..23d2fa1e005 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -430,6 +430,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #----------------------------- Logstash output --------------------------------- #output.logstash: @@ -506,6 +510,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. @@ -638,6 +646,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. @@ -735,6 +747,9 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index fec940b1219..bf6cf9dcd30 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -216,6 +216,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #----------------------------- Logstash output --------------------------------- #output.logstash: @@ -292,6 +296,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. @@ -424,6 +432,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. @@ -521,6 +533,9 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: diff --git a/libbeat/docs/shared-ssl-config.asciidoc b/libbeat/docs/shared-ssl-config.asciidoc index 6b0b1f6795f..2aa6f3b8110 100644 --- a/libbeat/docs/shared-ssl-config.asciidoc +++ b/libbeat/docs/shared-ssl-config.asciidoc @@ -157,3 +157,12 @@ The following elliptic curve types are available: * P-384 * P-521 +[float] +==== `renegotiation` + +This configures what types of TLS renegotiation are supported. The valid options +are `never`, `once`, and `freely`. The default value is never. + +* `never` - Disables renegotiation. +* `once` - Allows a remote server to request renegotiation once per connection. +* `freely` - Allows a remote server to repeatedly request renegotiation. diff --git a/libbeat/outputs/tls.go b/libbeat/outputs/tls.go index 0d601f365f7..8bb8f3eaf0e 100644 --- a/libbeat/outputs/tls.go +++ b/libbeat/outputs/tls.go @@ -36,6 +36,7 @@ type TLSConfig struct { CAs []string `config:"certificate_authorities"` Certificate CertificateConfig `config:",inline"` CurveTypes []tlsCurveType `config:"curve_types"` + Renegotiation tlsRenegotiationSupport `config:"renegotiation"` } type CertificateConfig struct { @@ -48,6 +49,8 @@ type tlsCipherSuite uint16 type tlsCurveType tls.CurveID +type tlsRenegotiationSupport tls.RenegotiationSupport + var tlsCipherSuites = map[string]tlsCipherSuite{ "ECDHE-ECDSA-AES-128-CBC-SHA": tlsCipherSuite(tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA), "ECDHE-ECDSA-AES-128-GCM-SHA256": tlsCipherSuite(tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256), @@ -74,6 +77,12 @@ var tlsCurveTypes = map[string]tlsCurveType{ "P-521": tlsCurveType(tls.CurveP521), } +var tlsRenegotiationSupportTypes = map[string]tlsRenegotiationSupport{ + "never": tlsRenegotiationSupport(tls.RenegotiateNever), + "once": tlsRenegotiationSupport(tls.RenegotiateOnceAsClient), + "freely": tlsRenegotiationSupport(tls.RenegotiateFreelyAsClient), +} + func (c *TLSConfig) Validate() error { hasCertificate := c.Certificate.Certificate != "" hasKey := c.Certificate.Key != "" @@ -144,6 +153,7 @@ func LoadTLSConfig(config *TLSConfig) (*transport.TLSConfig, error) { RootCAs: cas, CipherSuites: cipherSuites, CurvePreferences: curves, + Renegotiation: tls.RenegotiationSupport(config.Renegotiation), }, nil } @@ -289,3 +299,13 @@ func (ct *tlsCurveType) Unpack(s string) error { *ct = t return nil } + +func (r *tlsRenegotiationSupport) Unpack(s string) error { + t, found := tlsRenegotiationSupportTypes[s] + if !found { + return fmt.Errorf("invalid tls renegotiation type '%v'", s) + } + + *r = t + return nil +} diff --git a/libbeat/outputs/tls_test.go b/libbeat/outputs/tls_test.go index 59013c98ddf..4c8ce1eae49 100644 --- a/libbeat/outputs/tls_test.go +++ b/libbeat/outputs/tls_test.go @@ -85,6 +85,7 @@ func TestValuesSet(t *testing.T) { supported_protocols: [TLSv1.1, TLSv1.2] curve_types: - P-521 + renegotiation: freely `) if err != nil { @@ -100,6 +101,9 @@ func TestValuesSet(t *testing.T) { []transport.TLSVersion{transport.TLSVersion11, transport.TLSVersion12}, cfg.Versions) assert.Len(t, cfg.CurveTypes, 1) + assert.Equal(t, + tls.RenegotiateFreelyAsClient, + tls.RenegotiationSupport(cfg.Renegotiation)) } func TestApplyEmptyConfig(t *testing.T) { @@ -169,6 +173,10 @@ func TestCertificateFails(t *testing.T) { "unknown curve type", "curve_types: ['unknown curve type']", }, + { + "unknown renegotiation type", + "renegotiation: always", + }, } for i, test := range tests { diff --git a/libbeat/outputs/transport/tls.go b/libbeat/outputs/transport/tls.go index 2e00b3f77be..c0eb074f498 100644 --- a/libbeat/outputs/transport/tls.go +++ b/libbeat/outputs/transport/tls.go @@ -39,6 +39,10 @@ type TLSConfig struct { // Types of elliptic curves that will be used in an ECDHE handshake. If empty, // the implementation will choose a default. CurvePreferences []tls.CurveID + + // Renegotiation controls what types of renegotiation are supported. + // The default, never, is correct for the vast majority of applications. + Renegotiation tls.RenegotiationSupport } type TLSVersion uint16 diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 578d06b4478..eedcac9e35b 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -660,6 +660,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #----------------------------- Logstash output --------------------------------- #output.logstash: @@ -736,6 +740,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. @@ -868,6 +876,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. @@ -965,6 +977,9 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index a730f3ca170..1265829be34 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -668,6 +668,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #----------------------------- Logstash output --------------------------------- #output.logstash: @@ -744,6 +748,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. @@ -876,6 +884,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. @@ -973,6 +985,9 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 0a592543f46..cdbd759fc96 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -245,6 +245,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #----------------------------- Logstash output --------------------------------- #output.logstash: @@ -321,6 +325,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. @@ -453,6 +461,10 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. @@ -550,6 +562,9 @@ output.elasticsearch: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: From a5d5fbeb261cad7f69ffe3d0b304d372df90dd60 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 9 Oct 2017 14:25:16 +0200 Subject: [PATCH 83/96] Remove post-update command (#5306) The command was initially designed for the apm-server to modify the index pattern. But now a nicer solution was found and this is not needed anymore. --- libbeat/scripts/Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 0677719b5b3..52b2f68abfc 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -295,10 +295,6 @@ endif @mkdir -p $(PWD)/_meta/kibana/default/index-pattern @go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go -index '${BEAT_INDEX_PREFIX}-*' -beat-name ${BEAT_NAME} -beat-dir $(PWD) -version ${BEAT_VERSION} - # Runs the post-update target if it exists. Will not return errors - @-$(MAKE) post-update - - .PHONY: docs docs: ## @build Builds the documents for the beat sh ${ES_BEATS}/script/build_docs.sh ${BEAT_NAME} ${BEAT_PATH}/docs ${BUILD_DIR} From 67b664821977592ab23adb76a07b425a38e3b4bf Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 9 Oct 2017 14:50:28 +0200 Subject: [PATCH 84/96] Switch to Kibana 6.0.0-rc1 and increase retry to 300 (#5358) Already the compose timeout is set to 300. Now also setting the retries in the healtcheck to 300. --- metricbeat/module/kibana/_meta/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/_meta/Dockerfile b/metricbeat/module/kibana/_meta/Dockerfile index 39cecabb24e..3fd697cbc96 100644 --- a/metricbeat/module/kibana/_meta/Dockerfile +++ b/metricbeat/module/kibana/_meta/Dockerfile @@ -1,2 +1,2 @@ -FROM docker.elastic.co/kibana/kibana:6.0.0-beta1 -HEALTHCHECK --interval=1s --retries=120 CMD curl -f http://localhost:5601/api/status | grep '"disconnects"' +FROM docker.elastic.co/kibana/kibana:6.0.0-rc1 +HEALTHCHECK --interval=1s --retries=300 CMD curl -f http://localhost:5601/api/status | grep '"disconnects"' From 3ffa7c5be7f9b5710a675ea0370319c482f6f97b Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 9 Oct 2017 15:10:03 +0200 Subject: [PATCH 85/96] Fix RabbitMQ 5.x dashboard names (#5359) This were broken on case-sensitive file systems, because the name of the files didn't match the name of the IDs from the dashboards. Fixes #5277. --- CHANGELOG.asciidoc | 1 + ...ang-Process-Usage.json => RabbitMQ-Erlang-Process-Usage.json} | 0 .../{Rabbitmq-Memory-Usage.json => RabbitMQ-Memory-Usage.json} | 0 ...ndex-Operations.json => RabbitMQ-Queue-Index-Operations.json} | 0 4 files changed, 1 insertion(+) rename metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/{Rabbitmq-Erlang-Process-Usage.json => RabbitMQ-Erlang-Process-Usage.json} (100%) rename metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/{Rabbitmq-Memory-Usage.json => RabbitMQ-Memory-Usage.json} (100%) rename metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/{Rabbitmq-Queue-Index-Operations.json => RabbitMQ-Queue-Index-Operations.json} (100%) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 5934e2f3dfe..7decd66107d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -48,6 +48,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Change field type of http header from nested to object {pull}5258[5258] - Use `beat.name` instead of `beat.hostname` in the Host Overview dashboard. {pull}5340[5340] +- Fix the loading of 5.x dashboards. {issue}5277[5277] *Packetbeat* diff --git a/metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/Rabbitmq-Erlang-Process-Usage.json b/metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/RabbitMQ-Erlang-Process-Usage.json similarity index 100% rename from metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/Rabbitmq-Erlang-Process-Usage.json rename to metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/RabbitMQ-Erlang-Process-Usage.json diff --git a/metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/Rabbitmq-Memory-Usage.json b/metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/RabbitMQ-Memory-Usage.json similarity index 100% rename from metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/Rabbitmq-Memory-Usage.json rename to metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/RabbitMQ-Memory-Usage.json diff --git a/metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/Rabbitmq-Queue-Index-Operations.json b/metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/RabbitMQ-Queue-Index-Operations.json similarity index 100% rename from metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/Rabbitmq-Queue-Index-Operations.json rename to metricbeat/module/rabbitmq/_meta/kibana/5.x/visualization/RabbitMQ-Queue-Index-Operations.json From f9ac23fc9ac156739f9bd0be2c55945d0512088b Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 9 Oct 2017 18:36:00 +0200 Subject: [PATCH 86/96] Introduce jenkins_setup command to be reused (#5315) jenkins_setup in common.bash will also be used for the generator tests. --- dev-tools/common.bash | 18 ++++++++++++++++++ dev-tools/jenkins_ci.sh | 13 ++----------- dev-tools/jenkins_intake.sh | 15 +++------------ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/dev-tools/common.bash b/dev-tools/common.bash index aff8ba6b662..07dbe7d8130 100644 --- a/dev-tools/common.bash +++ b/dev-tools/common.bash @@ -78,3 +78,21 @@ setup_go_path() { debug "GOPATH=${GOPATH}" } + +jenkins_setup() { + : "${HOME:?Need to set HOME to a non-empty value.}" + : "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" + + # Setup Go. + export GOPATH=${WORKSPACE} + export PATH=${GOPATH}/bin:${PATH} + if [ -f ".go-version" ]; then + eval "$(gvm $(cat .go-version))" + else + eval "$(gvm 1.7.5)" + fi + + # Workaround for Python virtualenv path being too long. + export TEMP_PYTHON_ENV=$(mktemp -d) + export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env" +} diff --git a/dev-tools/jenkins_ci.sh b/dev-tools/jenkins_ci.sh index f89272ad437..9fe9de24c2b 100755 --- a/dev-tools/jenkins_ci.sh +++ b/dev-tools/jenkins_ci.sh @@ -12,18 +12,9 @@ if [ ! -d "$beat" ]; then exit fi -# Setup Go. -export GOPATH=${WORKSPACE} -export PATH=${GOPATH}/bin:${PATH} -if [ -f ".go-version" ]; then - eval "$(gvm $(cat .go-version))" -else - eval "$(gvm 1.7.5)" -fi +source ./dev-tools/common.bash -# Workaround for Python virtualenv path being too long. -TEMP_PYTHON_ENV=$(mktemp -d) -export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env" +jenkins_setup cleanup() { echo "Running cleanup..." diff --git a/dev-tools/jenkins_intake.sh b/dev-tools/jenkins_intake.sh index 9e23ea6a325..de8ec969ddb 100755 --- a/dev-tools/jenkins_intake.sh +++ b/dev-tools/jenkins_intake.sh @@ -4,18 +4,9 @@ set -euox pipefail : "${HOME:?Need to set HOME to a non-empty value.}" : "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" -# Setup Go. -export GOPATH=${WORKSPACE} -export PATH=${GOPATH}/bin:${PATH} -if [ -f ".go-version" ]; then - eval "$(gvm $(cat .go-version))" -else - eval "$(gvm 1.7.5)" -fi +source ./dev-tools/common.bash -# Workaround for Python virtualenv path being too long. -TEMP_PYTHON_ENV=$(mktemp -d) -export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env" +jenkins_setup cleanup() { echo "Running cleanup..." @@ -24,4 +15,4 @@ cleanup() { } trap cleanup EXIT -make check \ No newline at end of file +make check From 54a44810d605569b648e8850c63fe8261e5d10f3 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Mon, 9 Oct 2017 21:09:10 +0200 Subject: [PATCH 87/96] Add queue settings to docs (#4884) * Move queue setting to TOC + address some review * Remove flush_interval and spooler from outputconfig * remove 'slot' * add [float] tags * Update queueconfig.asciidoc --- auditbeat/docs/configuring-howto.asciidoc | 3 + filebeat/docs/configuring-howto.asciidoc | 3 + heartbeat/docs/configuring-howto.asciidoc | 3 + libbeat/docs/outputconfig.asciidoc | 27 ++++---- libbeat/docs/queueconfig.asciidoc | 74 ++++++++++++++++++++++ metricbeat/docs/configuring-howto.asciidoc | 3 + packetbeat/docs/configuring-howto.asciidoc | 3 + winlogbeat/docs/configuring-howto.asciidoc | 3 + 8 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 libbeat/docs/queueconfig.asciidoc diff --git a/auditbeat/docs/configuring-howto.asciidoc b/auditbeat/docs/configuring-howto.asciidoc index 54a63e86ce4..e16447de3cf 100644 --- a/auditbeat/docs/configuring-howto.asciidoc +++ b/auditbeat/docs/configuring-howto.asciidoc @@ -22,6 +22,7 @@ The following topics describe how to configure {beatname_uc}: * <> * <> * <<{beatname_lc}-configuration-reloading>> +* <> * <> * <> * <> @@ -46,6 +47,8 @@ include::./auditbeat-general-options.asciidoc[] include::./reload-configuration.asciidoc[] :allplatforms: +include::../../libbeat/docs/queueconfig.asciidoc[] + include::../../libbeat/docs/outputconfig.asciidoc[] include::../../libbeat/docs/shared-ssl-config.asciidoc[] diff --git a/filebeat/docs/configuring-howto.asciidoc b/filebeat/docs/configuring-howto.asciidoc index 3a2d4d2f4a3..4f6e53fb691 100644 --- a/filebeat/docs/configuring-howto.asciidoc +++ b/filebeat/docs/configuring-howto.asciidoc @@ -24,6 +24,7 @@ The following topics describe how to configure Filebeat: * <> * <> * <> +* <> * <> * <> * <> @@ -53,6 +54,8 @@ include::./filebeat-general-options.asciidoc[] include::./reload-configuration.asciidoc[] :allplatforms: +include::../../libbeat/docs/queueconfig.asciidoc[] + include::../../libbeat/docs/outputconfig.asciidoc[] include::../../libbeat/docs/shared-ssl-config.asciidoc[] diff --git a/heartbeat/docs/configuring-howto.asciidoc b/heartbeat/docs/configuring-howto.asciidoc index a46909a50f9..3774c62e983 100644 --- a/heartbeat/docs/configuring-howto.asciidoc +++ b/heartbeat/docs/configuring-howto.asciidoc @@ -23,6 +23,7 @@ The following topics describe how to configure Heartbeat: * <> * <> +* <> * <> * <> * <> @@ -43,6 +44,8 @@ include::./heartbeat-options.asciidoc[] include::./heartbeat-general-options.asciidoc[] :allplatforms: +include::../../libbeat/docs/queueconfig.asciidoc[] + include::../../libbeat/docs/outputconfig.asciidoc[] include::../../libbeat/docs/shared-ssl-config.asciidoc[] diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index cd81954c5eb..06f42b2c12d 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -281,10 +281,9 @@ However big batch sizes can also increase processing times, which might result i API errors, killed connections, timed-out publishing requests, and, ultimately, lower throughput. -Setting `bulk_max_size` to values less than or equal to 0 disables buffering in libbeat. When buffering is disabled, -Beats that publish single events (such as Packetbeat) send each event directly to -Elasticsearch. Beats that publish data in batches (such as Filebeat) send events in batches based on the -spooler size. +Setting `bulk_max_size` to values less than or equal to 0 disables the +splitting of batches. When splitting is disabled, the queue decides on the +number of events to be contained in a batch. ===== `timeout` @@ -526,10 +525,10 @@ However big batch sizes can also increase processing times, which might result i API errors, killed connections, timed-out publishing requests, and, ultimately, lower throughput. -Setting `bulk_max_size` to values less than or equal to 0 disables buffering in libbeat. When buffering is disabled, -Beats that publish single events (such as Packetbeat) send each event directly to -Elasticsearch. Beats that publish data in batches (such as Filebeat) send events in batches based on the -spooler size. +Setting `bulk_max_size` to values less than or equal to 0 disables the +splitting of batches. When splitting is disabled, the queue decides on the +number of events to be contained in a batch. + ===== `slow_start` @@ -930,11 +929,9 @@ of sending events. However big batch sizes can also increase processing times, which might result in API errors, killed connections, timed-out publishing requests, and, ultimately, lower throughput. -Setting `bulk_max_size` to values less than or equal to 0 disables buffering in -libbeat. When buffering is disabled, Beats that publish single events (such as -Packetbeat) send each event directly to Redis. Beats that publish -data in batches (such as Filebeat) send events in batches based on the spooler -size. +Setting `bulk_max_size` to values less than or equal to 0 disables the +splitting of batches. When splitting is disabled, the queue decides on the +number of events to be contained in a batch. ===== `ssl` @@ -1064,7 +1061,9 @@ The maximum number of events to buffer internally during publishing. The default Specifying a larger batch size may add some latency and buffering during publishing. However, for Console output, this setting does not affect how events are published. -Setting `bulk_max_size` to 0 disables buffering in libbeat. +Setting `bulk_max_size` to values less than or equal to 0 disables the +splitting of batches. When splitting is disabled, the queue decides on the +number of events to be contained in a batch. [[configuration-output-codec]] === Configure the output codec diff --git a/libbeat/docs/queueconfig.asciidoc b/libbeat/docs/queueconfig.asciidoc new file mode 100644 index 00000000000..3c4f77f4c74 --- /dev/null +++ b/libbeat/docs/queueconfig.asciidoc @@ -0,0 +1,74 @@ +[[configuring-internal-queue]] +== Configure the internal queue + +{beatname_uc} uses an internal queue to store events before publishing them. The +queue is responsible for buffering and combining events into batches that can +be consumed by the outputs. The outputs will use bulk operations to send a +batch of events in one transaction. + +You can configure the type and behavior of the internal queue by setting options in the `queue` section of the +{beatname_lc}.yml+ config file. + + +Example configuration: + +[source,yaml] +------------------------------------------------------------------------------ +queue.mem: + events: 4096 +------------------------------------------------------------------------------ + +[float] +[[configuration-internal-queue-memory]] +=== Configure the memory qeueue + +The memory queue keeps all events in memory. It is the only queue type +supported right now. By default no flush interval is configured. All events +published to this queue will be directly consumed by the outputs. +The output's `bulk_max_size` setting limits the number of events being processed at once. + +The memory queue waits for the output to acknowledge or drop events. If +the queue is full, no new events can be inserted into the memeory queue. Only +after the signal from the output will the queue free up space for more events to be accepted. + +To enforce spooling in the queue, set the `flush.min_events` and `flush.timeout` options. + +This sample configuration forwards events to the output if 512 events are +available or the oldest available event is already waiting for 5s in the queue: + +[source,yaml] +------------------------------------------------------------------------------ +queue.mem: + events: 4096 + flush.min_events: 512 + flush.timeout: 5s +------------------------------------------------------------------------------ + +[float] +==== Configuration options + +You can specify the following options in the `queue.mem` section of the +{beatname_lc}.yml+ config file: + +[float] +===== `events` + +Number of events the queue can store. + +The default value is 4096 events. + +[float] +===== `flush.min_events` + +Minimum number of events required for publishing. If this value is set to 0, the +output can start publishing events without additional waiting times. Otherwise +the output has to wait for more events to become available. + +The default value is 0. + +[float] +===== `flush.timeout` + +Maximum wait time for `flush.min_events` to be fulfilled. If set to 0s, events +will be immediately available for consumption. + +The default values is 0s. + diff --git a/metricbeat/docs/configuring-howto.asciidoc b/metricbeat/docs/configuring-howto.asciidoc index 27fdf740058..2df113c9763 100644 --- a/metricbeat/docs/configuring-howto.asciidoc +++ b/metricbeat/docs/configuring-howto.asciidoc @@ -22,6 +22,7 @@ The following topics describe how to configure Metricbeat: * <> * <> * <> +* <> * <> * <> * <> @@ -44,6 +45,8 @@ include::./metricbeat-general-options.asciidoc[] include::./reload-configuration.asciidoc[] :allplatforms: +include::../../libbeat/docs/queueconfig.asciidoc[] + include::../../libbeat/docs/outputconfig.asciidoc[] include::../../libbeat/docs/shared-ssl-config.asciidoc[] diff --git a/packetbeat/docs/configuring-howto.asciidoc b/packetbeat/docs/configuring-howto.asciidoc index bab608cf2c2..018480bf84d 100644 --- a/packetbeat/docs/configuring-howto.asciidoc +++ b/packetbeat/docs/configuring-howto.asciidoc @@ -24,6 +24,7 @@ The following topics describe how to configure Packetbeat: * <> * <> * <> +* <> * <> * <> * <> @@ -44,6 +45,8 @@ include::./packetbeat-options.asciidoc[] include::./packetbeat-general-options.asciidoc[] :allplatforms: +include::../../libbeat/docs/queueconfig.asciidoc[] + include::../../libbeat/docs/outputconfig.asciidoc[] include::../../libbeat/docs/shared-ssl-config.asciidoc[] diff --git a/winlogbeat/docs/configuring-howto.asciidoc b/winlogbeat/docs/configuring-howto.asciidoc index 8a6fa05ebce..dd1e94d2748 100644 --- a/winlogbeat/docs/configuring-howto.asciidoc +++ b/winlogbeat/docs/configuring-howto.asciidoc @@ -19,6 +19,7 @@ The following topics describe how to configure Winlogbeat: * <> * <> +* <> * <> * <> * <> @@ -38,6 +39,8 @@ include::./winlogbeat-options.asciidoc[] include::./winlogbeat-general-options.asciidoc[] :win: +include::../../libbeat/docs/queueconfig.asciidoc[] + include::../../libbeat/docs/outputconfig.asciidoc[] include::../../libbeat/docs/shared-ssl-config.asciidoc[] From 4f13672080a076e23e92804bf57be874b03cf97e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Tue, 10 Oct 2017 10:12:42 +0200 Subject: [PATCH 88/96] Fix Test_Rotator's segmentation fault --- libbeat/logp/file_rotator_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libbeat/logp/file_rotator_test.go b/libbeat/logp/file_rotator_test.go index 10d8519d666..b03e7bd0793 100644 --- a/libbeat/logp/file_rotator_test.go +++ b/libbeat/logp/file_rotator_test.go @@ -27,12 +27,13 @@ func Test_Rotator(t *testing.T) { rotateeverybytes := uint64(1000) keepfiles := 3 - + perms := uint32(0655) rotator := FileRotator{ Path: dir, Name: "packetbeat", RotateEveryBytes: &rotateeverybytes, KeepFiles: &keepfiles, + Permissions: &perms, } err = rotator.Rotate() @@ -120,12 +121,14 @@ func Test_Rotator_By_Bytes(t *testing.T) { rotateeverybytes := uint64(100) keepfiles := 3 + perms := uint32(0655) rotator := FileRotator{ Path: dir, Name: "packetbeat", RotateEveryBytes: &rotateeverybytes, KeepFiles: &keepfiles, + Permissions: &perms, } for i := 0; i < 300; i++ { @@ -189,12 +192,14 @@ func TestRaceConditions(t *testing.T) { rotateeverybytes := uint64(10) keepfiles := 20 + perms := uint32(0600) rotator := FileRotator{ Path: dir, Name: "testbeat", RotateEveryBytes: &rotateeverybytes, KeepFiles: &keepfiles, + Permissions: &perms, } for i := 0; i < 1000; i++ { From 1a98c8022c6465a3bf012fc3a0b01de31e517e0c Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 10 Oct 2017 11:05:47 +0200 Subject: [PATCH 89/96] Only read go version from .go-version file (#5367) Remove default golang version as not needed anymore. All repos / branches have now a .go-version file. --- dev-tools/common.bash | 6 +----- dev-tools/jenkins_ci.ps1 | 8 ++------ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/dev-tools/common.bash b/dev-tools/common.bash index 07dbe7d8130..048c835f2c0 100644 --- a/dev-tools/common.bash +++ b/dev-tools/common.bash @@ -86,11 +86,7 @@ jenkins_setup() { # Setup Go. export GOPATH=${WORKSPACE} export PATH=${GOPATH}/bin:${PATH} - if [ -f ".go-version" ]; then - eval "$(gvm $(cat .go-version))" - else - eval "$(gvm 1.7.5)" - fi + eval "$(gvm $(cat .go-version))" # Workaround for Python virtualenv path being too long. export TEMP_PYTHON_ENV=$(mktemp -d) diff --git a/dev-tools/jenkins_ci.ps1 b/dev-tools/jenkins_ci.ps1 index a7f9f5d37a4..10a28ae5650 100755 --- a/dev-tools/jenkins_ci.ps1 +++ b/dev-tools/jenkins_ci.ps1 @@ -15,11 +15,7 @@ function Exec # Setup Go. $env:GOPATH = $env:WORKSPACE $env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH" -if (Test-Path -PathType leaf .go-version) { - & gvm --format=powershell $(Get-Content .go-version) | Invoke-Expression -} else { - & gvm --format=powershell 1.7.5 | Invoke-Expression -} +& gvm --format=powershell $(Get-Content .go-version) | Invoke-Expression if (Test-Path "$env:beat") { cd "$env:beat" @@ -58,4 +54,4 @@ echo "System testing $env:beat" exec { go get github.com/docker/libcompose } exec { go test -race -c -cover -covermode=atomic -coverpkg ./... } exec { cd tests/system } -exec { nosetests --with-timer --with-xunit --xunit-file=../../build/TEST-system.xml } "System test FAILURE" \ No newline at end of file +exec { nosetests --with-timer --with-xunit --xunit-file=../../build/TEST-system.xml } "System test FAILURE" From 483982b9a16f453ce01f413dfe91880d25632671 Mon Sep 17 00:00:00 2001 From: Mahmoud Saada Date: Tue, 10 Oct 2017 11:11:30 -0400 Subject: [PATCH 90/96] Add Azure VM support for add_cloud_metadata processor (#5355) * Add Azure VM Cloud Metadata Support * Update CHANGELOG.asciidoc * Update processors-using.asciidoc --- CHANGELOG.asciidoc | 1 + libbeat/docs/processors-using.asciidoc | 19 +++++ .../add_cloud_metadata/add_cloud_metadata.go | 5 ++ .../add_cloud_metadata/provider_azure_vm.go | 25 ++++++ .../provider_azure_vm_test.go | 77 +++++++++++++++++++ 5 files changed, 127 insertions(+) create mode 100644 libbeat/processors/add_cloud_metadata/provider_azure_vm.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 7decd66107d..a76d6eb4716 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -63,6 +63,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051] - Fix reloader error message to only print on actual error {pull}5066[5066] - Add support for enabling TLS renegotiation. {issue}4386[4386] +- Add Azure VM support for add_cloud_metadata processor {pull}5355[5355] *Auditbeat* diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 22264095793..eca64a6715f 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -260,6 +260,7 @@ The following cloud providers are supported: - Google Compute Engine (GCE) - https://www.qcloud.com/?lang=en[Tencent Cloud] (QCloud) - Alibaba Cloud (ECS) +- Azure Virtual Machine The simple configuration below enables the processor. @@ -365,6 +366,24 @@ ECS instance. } ------------------------------------------------------------------------------- +_Azure Virtual Machine_ + +[source,json] +------------------------------------------------------------------------------- +{ + "meta": { + "cloud": { + "provider": "az", + "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", + "instance_name": "test-az-vm", + "machine_type": "Standard_D3_v2", + "region": "eastus2" + } + } +} +------------------------------------------------------------------------------- + + [[add-locale]] === Add the local time zone diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index 5b1424fa029..9d55e3c99f6 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -260,6 +260,10 @@ func setupFetchers(c *common.Config) ([]*metadataFetcher, error) { if err != nil { return fetchers, err } + azFetcher, err := newAzureVmMetadataFetcher(c) + if err != nil { + return fetchers, err + } fetchers = []*metadataFetcher{ doFetcher, @@ -267,6 +271,7 @@ func setupFetchers(c *common.Config) ([]*metadataFetcher, error) { gceFetcher, qcloudFetcher, ecsFetcher, + azFetcher, } return fetchers, nil } diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go new file mode 100644 index 00000000000..10193d9cd4c --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -0,0 +1,25 @@ +package add_cloud_metadata + +import ( + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" +) + +// Azure VM Metadata Service +func newAzureVmMetadataFetcher(config *common.Config) (*metadataFetcher, error) { + azMetadataURI := "/metadata/instance/compute?api-version=2017-04-02" + azHeaders := map[string]string{"Metadata": "true"} + azSchema := func(m map[string]interface{}) common.MapStr { + out, _ := s.Schema{ + "instance_id": c.Str("vmId"), + "instance_name": c.Str("name"), + "machine_type": c.Str("vmSize"), + "region": c.Str("location"), + }.Apply(m) + return out + } + + fetcher, err := newMetadataFetcher(config, "az", azHeaders, metadataHost, azSchema, azMetadataURI) + return fetcher, err +} diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go new file mode 100644 index 00000000000..ddfe6590357 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -0,0 +1,77 @@ +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +const azInstanceIdentityDocument = `{ + "location": "eastus2", + "name": "test-az-vm", + "offer": "UbuntuServer", + "osType": "Linux", + "platformFaultDomain": "0", + "platformUpdateDomain": "0", + "publisher": "Canonical", + "sku": "14.04.4-LTS", + "version": "14.04.201605091", + "vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", + "vmSize": "Standard_D3_v2" +}` + +func initAzureTestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/metadata/instance/compute?api-version=2017-04-02" && r.Header.Get("Metadata") == "true" { + w.Write([]byte(azInstanceIdentityDocument)) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveAzureMetadata(t *testing.T) { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + server := initAzureTestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + }) + if err != nil { + t.Fatal(err) + } + + p, err := newCloudMetadata(config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(&beat.Event{Fields: common.MapStr{}}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "meta": common.MapStr{ + "cloud": common.MapStr{ + "provider": "az", + "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", + "instance_name": "test-az-vm", + "machine_type": "Standard_D3_v2", + "region": "eastus2", + }, + }, + } + assert.Equal(t, expected, actual.Fields) +} From d6c118c98f20908d733f2c029f61112e9ee4986b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Fri, 7 Jul 2017 18:30:23 +0200 Subject: [PATCH 91/96] Add permissions configuration for file output --- libbeat/outputs/fileout/config.go | 1 + libbeat/outputs/fileout/file.go | 4 ++++ metricbeat/metricbeat.reference.yml | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index 9e26a19cd21..26553f07d0e 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -13,6 +13,7 @@ type config struct { RotateEveryKb int `config:"rotate_every_kb" validate:"min=1"` NumberOfFiles int `config:"number_of_files"` Codec codec.Config `config:"codec"` + Permissions uint32 `config:"permissions"` } var ( diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index bf569058f39..6b23f1c3455 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -47,6 +47,9 @@ func (out *fileOutput) init(beat beat.Info, config config) error { out.rotator.Path = config.Path out.rotator.Name = config.Filename + if config.Permissions != 0 { + out.rotator.Permissions = &config.Permissions + } if out.rotator.Name == "" { out.rotator.Name = out.beat.Beat } @@ -58,6 +61,7 @@ func (out *fileOutput) init(beat beat.Info, config config) error { out.codec = enc + logp.Info("File output permissions set to: %v", *out.rotator.Permissions) logp.Info("File output path set to: %v", out.rotator.Path) logp.Info("File output base filename set to: %v", out.rotator.Name) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index eedcac9e35b..2352b74baca 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1004,6 +1004,10 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. + # default is 0600. + #permissions: 0666 + #----------------------------- Console output --------------------------------- #output.console: From f98de959bcc2385f40827cb32b853f4899599def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Mon, 10 Jul 2017 10:25:21 +0200 Subject: [PATCH 92/96] Add permissions configuration for file output --- libbeat/docs/outputconfig.asciidoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 06f42b2c12d..eca50f8c49f 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -976,6 +976,7 @@ output.file: filename: {beatname_lc} #rotate_every_kb: 10000 #number_of_files: 7 + #permissions: 0666 ------------------------------------------------------------------------------ ==== Configuration options @@ -1011,6 +1012,10 @@ The maximum number of files to save under <>. When this number of f oldest file is deleted, and the rest of the files are shifted from last to first. The default is 7 files. +===== `permissions` + +Permissions to use for file creation. The default is 0600. + ===== `codec` Output codec configuration. If the `codec` section is missing, events will be json encoded. From 8876ded69fa1092bfabfcacabe725b003b21cd68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Mon, 10 Jul 2017 10:26:59 +0200 Subject: [PATCH 93/96] Small change --- metricbeat/metricbeat.reference.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 2352b74baca..bae61040079 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1004,8 +1004,7 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 - # Permissions to use for file creation. - # default is 0600. + # Permissions to use for file creation. The default is 0600. #permissions: 0666 From d825830d31d6e0f6adb6f2fa41ea977b09a9e085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Wed, 19 Jul 2017 10:49:19 +0200 Subject: [PATCH 94/96] Change config.reference.yml appropriatly & fix wrong value in docs --- auditbeat/auditbeat.reference.yml | 3 +++ filebeat/filebeat.reference.yml | 3 +++ heartbeat/heartbeat.reference.yml | 3 +++ libbeat/_meta/config.reference.yml | 3 +++ libbeat/docs/outputconfig.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 2 +- packetbeat/packetbeat.reference.yml | 3 +++ winlogbeat/winlogbeat.reference.yml | 3 +++ 8 files changed, 20 insertions(+), 2 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 42920c139c4..e1c8df1ebbe 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -625,6 +625,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index dee3140514f..23f5f4d9456 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1045,6 +1045,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 23d2fa1e005..248f10c9aad 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -774,6 +774,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index bf6cf9dcd30..a7d1196d052 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -560,6 +560,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index eca50f8c49f..979a235319a 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -976,7 +976,7 @@ output.file: filename: {beatname_lc} #rotate_every_kb: 10000 #number_of_files: 7 - #permissions: 0666 + #permissions: 0600 ------------------------------------------------------------------------------ ==== Configuration options diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index bae61040079..47aa495d7ad 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1005,7 +1005,7 @@ output.elasticsearch: #number_of_files: 7 # Permissions to use for file creation. The default is 0600. - #permissions: 0666 + #permissions: 0600 #----------------------------- Console output --------------------------------- diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 1265829be34..e68bea23494 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1012,6 +1012,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index cdbd759fc96..7382c9eba4d 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -589,6 +589,9 @@ output.elasticsearch: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: From 8227731ba16a1b531637637ec93ee36daced1f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Wed, 19 Jul 2017 11:29:02 +0200 Subject: [PATCH 95/96] Fix bug when no value given for permissions & add proper default value to config.go and file_rotator.go --- libbeat/logp/file_rotator.go | 15 +++++++-------- libbeat/outputs/fileout/config.go | 1 + libbeat/outputs/fileout/file.go | 7 +++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/libbeat/logp/file_rotator.go b/libbeat/logp/file_rotator.go index c7db4d7acd6..b528618d898 100644 --- a/libbeat/logp/file_rotator.go +++ b/libbeat/logp/file_rotator.go @@ -12,6 +12,7 @@ import ( const RotatorMaxFiles = 1024 const DefaultKeepFiles = 7 const DefaultRotateEveryBytes = 10 * 1024 * 1024 +const DefaultPermissions = 0600 type FileRotator struct { Path string @@ -56,6 +57,11 @@ func (rotator *FileRotator) CheckIfConfigSane() error { *rotator.RotateEveryBytes = DefaultRotateEveryBytes } + if rotator.Permissions == nil { + rotator.Permissions = new(uint32) + *rotator.Permissions = DefaultPermissions + } + if *rotator.KeepFiles < 2 || *rotator.KeepFiles >= RotatorMaxFiles { return fmt.Errorf("the number of files to keep should be between 2 and %d", RotatorMaxFiles-1) } @@ -164,7 +170,7 @@ func (rotator *FileRotator) Rotate() error { // create the new file path := rotator.FilePath(0) - current, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(rotator.getPermissions())) + current, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(*rotator.Permissions)) if err != nil { return err } @@ -177,10 +183,3 @@ func (rotator *FileRotator) Rotate() error { return nil } - -func (rotator *FileRotator) getPermissions() uint32 { - if rotator.Permissions == nil { - return 0600 - } - return *rotator.Permissions -} diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index 26553f07d0e..a970e2025e6 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -20,6 +20,7 @@ var ( defaultConfig = config{ NumberOfFiles: 7, RotateEveryKb: 10 * 1024, + Permissions: 0600, } ) diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index 6b23f1c3455..42f64620407 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -47,9 +47,6 @@ func (out *fileOutput) init(beat beat.Info, config config) error { out.rotator.Path = config.Path out.rotator.Name = config.Filename - if config.Permissions != 0 { - out.rotator.Permissions = &config.Permissions - } if out.rotator.Name == "" { out.rotator.Name = out.beat.Beat } @@ -61,10 +58,12 @@ func (out *fileOutput) init(beat beat.Info, config config) error { out.codec = enc - logp.Info("File output permissions set to: %v", *out.rotator.Permissions) logp.Info("File output path set to: %v", out.rotator.Path) logp.Info("File output base filename set to: %v", out.rotator.Name) + logp.Info("File output permissions set to: %#o", config.Permissions) + out.rotator.Permissions = &config.Permissions + rotateeverybytes := uint64(config.RotateEveryKb) * 1024 logp.Info("Rotate every bytes set to: %v", rotateeverybytes) out.rotator.RotateEveryBytes = &rotateeverybytes From 9a136fb7e23b63cb7d2b615e513b656bef8d76bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ry=20Plassat?= Date: Tue, 10 Oct 2017 10:12:42 +0200 Subject: [PATCH 96/96] Fix Test_Rotator's segmentation fault --- libbeat/logp/file_rotator_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libbeat/logp/file_rotator_test.go b/libbeat/logp/file_rotator_test.go index 10d8519d666..b03e7bd0793 100644 --- a/libbeat/logp/file_rotator_test.go +++ b/libbeat/logp/file_rotator_test.go @@ -27,12 +27,13 @@ func Test_Rotator(t *testing.T) { rotateeverybytes := uint64(1000) keepfiles := 3 - + perms := uint32(0655) rotator := FileRotator{ Path: dir, Name: "packetbeat", RotateEveryBytes: &rotateeverybytes, KeepFiles: &keepfiles, + Permissions: &perms, } err = rotator.Rotate() @@ -120,12 +121,14 @@ func Test_Rotator_By_Bytes(t *testing.T) { rotateeverybytes := uint64(100) keepfiles := 3 + perms := uint32(0655) rotator := FileRotator{ Path: dir, Name: "packetbeat", RotateEveryBytes: &rotateeverybytes, KeepFiles: &keepfiles, + Permissions: &perms, } for i := 0; i < 300; i++ { @@ -189,12 +192,14 @@ func TestRaceConditions(t *testing.T) { rotateeverybytes := uint64(10) keepfiles := 20 + perms := uint32(0600) rotator := FileRotator{ Path: dir, Name: "testbeat", RotateEveryBytes: &rotateeverybytes, KeepFiles: &keepfiles, + Permissions: &perms, } for i := 0; i < 1000; i++ {