From dc6c2a10e99570690d24497304c76e73c1d3bfe4 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Fri, 20 Nov 2015 18:58:56 +0100 Subject: [PATCH 1/4] Update to most recent libbeat version and shared files --- Godeps/Godeps.json | 76 +++++++++---------- .../elastic/libbeat/service/service.go | 7 +- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ea7d10dfd2ac..4d2c914f134c 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -7,98 +7,98 @@ "Deps": [ { "ImportPath": "github.com/elastic/libbeat/beat", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/cfgfile", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/common", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/logp", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/outputs", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/publisher", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/service", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/cfgfile", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/common", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/logp", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/outputs", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/publisher", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/service", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/cfgfile", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/common", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/logp", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/outputs", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/publisher", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/elastic/libbeat/service", - "Comment": "v1.0.0-rc1-68-g5313047", - "Rev": "531304787e67d86f958ca96e2e1516c4f1bc7c5c" + "Comment": "v1.0.0-rc2-3-ga35e184", + "Rev": "a35e1840d08381622fb4c428f8e1b122c0b42b29" }, { "ImportPath": "github.com/garyburd/redigo/internal", diff --git a/Godeps/_workspace/src/github.com/elastic/libbeat/service/service.go b/Godeps/_workspace/src/github.com/elastic/libbeat/service/service.go index fcc45a090a99..533b4cd73ed0 100644 --- a/Godeps/_workspace/src/github.com/elastic/libbeat/service/service.go +++ b/Godeps/_workspace/src/github.com/elastic/libbeat/service/service.go @@ -7,6 +7,7 @@ import ( "os/signal" "runtime" "runtime/pprof" + "sync" "syscall" "github.com/elastic/libbeat/logp" @@ -16,19 +17,21 @@ import ( // The stopFunction should break the loop in the Beat so that // the service shut downs gracefully. func HandleSignals(stopFunction func()) { + var callback sync.Once + // On ^C or SIGTERM, gracefully stop the sniffer sigc := make(chan os.Signal, 1) signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) go func() { <-sigc logp.Debug("service", "Received sigterm/sigint, stopping") - stopFunction() + callback.Do(stopFunction) }() // Handle the Windows service events go ProcessWindowsControlEvents(func() { logp.Debug("service", "Received svc stop/shutdown request") - stopFunction() + callback.Do(stopFunction) }) } From 5639d0a3e6bfefc7a8e9cce1e93e419debb82c7f Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Fri, 20 Nov 2015 19:30:18 +0100 Subject: [PATCH 2/4] Closed changelog and set version --- CHANGELOG.md | 12 +++++++++++- main.go | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e82be60342d..ba131a198b8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,17 @@ All notable changes to this project will be documented in this file based on the [Keep a Changelog](http://keepachangelog.com/) Standard. -## [Unreleased](https://github.com/elastic/libbeat/compare/1.0.0-rc2...HEAD) +## [Unreleased](https://github.com/elastic/libbeat/compare/1.0.0...HEAD) + +### Backward Compatibility Breaks + +### Bugfixes + +### Added + +### Deprecated + +## [1.0.0](https://github.com/elastic/libbeat/compare/1.0.0-rc2...1.0.0) - 2015-11-24 ### Backward Compatibility Breaks diff --git a/main.go b/main.go index 100962c986e3..e26a31416c65 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,7 @@ import ( "github.com/elastic/libbeat/logp" ) -var Version = "1.0.0-rc2" +var Version = "1.0.0" var Name = "filebeat" var GlobalBeat *beat.Beat From 76f91e4657f08e845e0204e554c272147ffe83e7 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Fri, 20 Nov 2015 20:55:12 +0100 Subject: [PATCH 3/4] Fixed URLs --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba131a198b8e..f264c723d9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file based on the [Keep a Changelog](http://keepachangelog.com/) Standard. -## [Unreleased](https://github.com/elastic/libbeat/compare/1.0.0...HEAD) +## [Unreleased](https://github.com/elastic/filebeat/compare/1.0.0...HEAD) ### Backward Compatibility Breaks @@ -12,7 +12,7 @@ All notable changes to this project will be documented in this file based on the ### Deprecated -## [1.0.0](https://github.com/elastic/libbeat/compare/1.0.0-rc2...1.0.0) - 2015-11-24 +## [1.0.0](https://github.com/elastic/filebeat/compare/1.0.0-rc2...1.0.0) - 2015-11-24 ### Backward Compatibility Breaks @@ -24,7 +24,7 @@ All notable changes to this project will be documented in this file based on the ### Deprecated -## [1.0.0-rc2](https://github.com/elastic/libbeat/compare/1.0.0-rc1...1.0.0-rc2) - 2015-11-17 +## [1.0.0-rc2](https://github.com/elastic/filebeat/compare/1.0.0-rc1...1.0.0-rc2) - 2015-11-17 ### Backward Compatibility Breaks - Removed utf-16be-bom encoding support. Support will be added with fix for #205 @@ -42,7 +42,7 @@ All notable changes to this project will be documented in this file based on the ### Deprecated -## [1.0.0-rc1](https://github.com/elastic/libbeat/compare/1.0.0-beta4...1.0.0-rc1) +## [1.0.0-rc1](https://github.com/elastic/filebeat/compare/1.0.0-beta4...1.0.0-rc1) ### Backward Compatibility Breaks - Rename the timestamp field with @timestamp #168 From 6557682bbf63227d580b84196e6eed32f992b5e2 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Fri, 20 Nov 2015 13:29:27 -0800 Subject: [PATCH 4/4] adds edits for the migration topic and some misc fixes --- docs/command-line.asciidoc | 20 ++++++++-------- docs/configuration.asciidoc | 18 +++++++++------ docs/getting-started.asciidoc | 27 +++++++++++----------- docs/index.asciidoc | 4 ++-- docs/migration.asciidoc | 43 ++++++++++++++++++++--------------- docs/overview.asciidoc | 7 +++--- 6 files changed, 64 insertions(+), 55 deletions(-) diff --git a/docs/command-line.asciidoc b/docs/command-line.asciidoc index b54f386c67f0..97540235139e 100644 --- a/docs/command-line.asciidoc +++ b/docs/command-line.asciidoc @@ -13,21 +13,21 @@ configuration file. ---------------------------------------------------------------------------- $ ./filebeat -h Usage of ./filebeat: - -N Disable actual publishing for testing + -N Disable actual publishing for testing -c string - Configuration file (default "/etc/filebeat/filebeat.yml") + Configuration file (default "/etc/filebeat/filebeat.yml") + -configtest + Test configuration and exit. -cpuprofile string - Write cpu profile to file + Write cpu profile to file -d string - Enable certain debug selectors - -e Output to stdout and disable syslog/file output + Enable certain debug selectors + -e Log to stderr and disable syslog/file output -memprofile string - Write memory profile to this file - -test - Test configuration and exit. - -v Log at INFO level + Write memory profile to this file + -v Log at INFO level -version - Print version and exit + Print version and exit ----------------------------------------------------------------------------- diff --git a/docs/configuration.asciidoc b/docs/configuration.asciidoc index 3d1f9a638cf1..de4e6a9f7e3f 100644 --- a/docs/configuration.asciidoc +++ b/docs/configuration.asciidoc @@ -2,21 +2,25 @@ [[filebeat-configuration-details]] == Configuration Options +Before modifying configuration settings, make sure you've read <>. + The Filebeat configuration file uses http://yaml.org/[YAML] for its syntax. -The _etc/filebeat.yml_ file consists of the following sections: +The file contains config options common to all Beats. These options are described +in the following sections of the {libbeat}/configuration.html[Beats Platform Reference]: * {libbeat}/configuration.html#configuration-shipper[Shipper] * {libbeat}/configuration.html#configuration-output[Output] -* <> -* {libbeat}/configuration.html#configuration-logging[Logging (optional)] -* {libbeat}/configuration.html#configuration-run-options[Run options (optional)] +* {libbeat}/configuration.html#configuration-logging[Logging (Optional)] +* {libbeat}/configuration.html#configuration-run-options[Run Options (Optional)] -The `shipper`, `output`, `logging`, and `runoptions` sections contain configuration options that are common to all Beats, so they are covered in the {libbeat}/configuration.html[Beats Platform Reference]. +The Filebeat config options are described here: + +* <> [[configuration-filebeat-options]] -=== Prospectors +=== Filebeat -The `prospectors` section specifies a list of prospectors that Filebeat +The `filebeat` section specifies a list of `prospectors` that Filebeat uses to locate and process log files. Each prospector item begins with a dash (-) and specifies prospector-specific configuration options, including the list of paths that are crawled to locate log files. diff --git a/docs/getting-started.asciidoc b/docs/getting-started.asciidoc index cac386e08f68..f3fbbe42e9df 100644 --- a/docs/getting-started.asciidoc +++ b/docs/getting-started.asciidoc @@ -12,7 +12,7 @@ After you have installed these products, you can start <> [[filebeat-installation]] === Installing Filebeat -To download and install Filebeat, run the following commands. Use the commands that +To download and install Filebeat, run the following commands. Use the commands that work with your system (deb for Debian/Ubuntu, rpm for Redhat/Centos/Fedora, mac for OS X). deb: @@ -45,14 +45,14 @@ tar xzvf filebeat-{version}-darwin.tgz NOTE: We also provide 32-bit images that you can get from our https://www.elastic.co/downloads/beats/filebeat[download page]. -Before starting filebeat, you should look at the configuration options in the configuration -file, for example `/etc/filebeat/filebeat.yml`. For more information about these options, -see <>. +Before starting filebeat, you should look at the configuration options in the configuration +file, for example `/etc/filebeat/filebeat.yml`. For more information about these options, +see <>. [[filebeat-configuration]] === Configuring Filebeat -To configure Filebeat, you edit the _filebeat.yml_ file. Here is a sample of +To configure Filebeat, you edit the _filebeat.yml_ file. Here is a sample of the _filebeat.yml_ file: [source,yaml] @@ -77,8 +77,8 @@ filebeat: type: log ------------------------------------------------------------------------------------- -Filebeat uses predefined default values for most configuration options. For the most basic -Filebeat configuration, you can define a single prospector with a single path. For example: +Filebeat uses predefined default values for most configuration options. For the most basic +Filebeat configuration, you can define a single prospector with a single path. For example: [source,yaml] ------------------------------------------------------------------------------------- @@ -89,7 +89,7 @@ filebeat: - "/var/log/*.log" ------------------------------------------------------------------------------------- -The prospector in this example harvests all files in the path `/var/log/*.log`, which means +The prospector in this example harvests all files in the path `/var/log/*.log`, which means that Filebeat will harvest all files in the directory `/var/log/` that end with `.log`. All patterns supported by https://golang.org/pkg/path/filepath/#Glob[Golang Glob] are also supported here. @@ -118,11 +118,10 @@ filebeat: ------------------------------------------------------------------------------------- The config file in the example starts two prospectors. The first prospector has two harvesters, -one harvesting the `system.log` file, and the other harvesting `wifi.log`. The second prospector +one harvesting the `system.log` file, and the other harvesting `wifi.log`. The second prospector starts a harvester for each file in the apache directory. - -See <> for more details about each configuration option. +See <> for more details about each configuration option. [[filebeat-template]] === Loading the Index Template in Elasticsearch @@ -154,8 +153,8 @@ where `localhost:9200` is the IP and port where Elasticsearch is listening. === Setting Up Filebeat to Use Logstash -If you want to use Logstash to perform additional processing on the data collected by -Filebeat, you need to set up Filebeat to use Logstash. For detailed steps, see +If you want to use Logstash to perform additional processing on the data collected by +Filebeat, you need to set up Filebeat to use Logstash. For detailed steps, see {libbeat}/getting-started.html#logstash-setup[Setting Up Logstash]. @@ -184,6 +183,6 @@ mac: sudo ./filebeat -e -c filebeat.yml -d "publish" ---------------------------------------------------------------------- -Filebeat is now ready to send log files to your defined output. +Filebeat is now ready to send log files to your defined output. Enjoy! diff --git a/docs/index.asciidoc b/docs/index.asciidoc index c5f5af685bb7..b7aa69afd367 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -6,12 +6,12 @@ include::./overview.asciidoc[] include::./getting-started.asciidoc[] -include::./fields.asciidoc[] - include::./configuration.asciidoc[] include::./command-line.asciidoc[] +include::./fields.asciidoc[] + include::./migration.asciidoc[] include::./support.asciidoc[] diff --git a/docs/migration.asciidoc b/docs/migration.asciidoc index 904b441ea3e0..92f1e7c2dbf2 100644 --- a/docs/migration.asciidoc +++ b/docs/migration.asciidoc @@ -69,7 +69,7 @@ conversion to create a Filebeat config file for each Logstash Forwarder config f Before migrating your config files, we recommend that you first read the <> section to understand the Filebeat options. -NOTE: Logstash Forwarder has the option of auto completing environment variables in +NOTE: Logstash Forwarder has the option of autocompleting environment variables in the configuration file. This option currently doesn't exist in Filebeat. ==== Migrating the "files" Section @@ -341,27 +341,29 @@ The `config_dir` option specifies the path to the directory that contains additi |`-cpuProfileFile` | -|`cpuProfileFile` option was removed. You can use the profiling options of libbeat instead. For more details on profiling, see https://github.com/elastic/libbeat/issues/122. +|`cpuProfileFile` was removed. You can use the profiling options of libbeat instead. For more details on profiling, see https://github.com/elastic/libbeat/issues/122. |`-quiet` | -|The `quiet` option was removed. Libbeat is now used for logging, so you must use the libbeat {libbeat}/configuration.html#configuration-logging[logging options] instead. +|`quiet` was removed. Libbeat is now used for logging, so you must use the libbeat {libbeat}/configuration.html#configuration-logging[logging options] instead. |=== -=== Changes in the output fields +=== Changes to the Output Fields In the default configuration, Filebeat structures its output documents a little -different from the Logstash Forwarder. This section discusses the differences +differently from the Logstash Forwarder. This section discusses the differences and the options you have in case you want compatibility with the Logstash Forwarder. -One such difference is that the custom fields (added from the configuration -file) are set top level in Logstash Forwarder but are grouped together under a -`fields` dictionary in Filebeat. If you need the old behavior during the -migration phase, you can use the <> configuration option: +==== Custom Fields are Grouped Under a "fields" Dictionary + +The custom fields (added from the configuration file) are set as top-level +fields in Logstash Forwarder but are grouped together under a `fields` +dictionary in Filebeat. If you need the old behavior during the migration phase, +you can use the <> configuration option: [source,yaml] ------------------------------------------------------------------------------------- @@ -376,14 +378,17 @@ filebeat: fields_under_root: true ------------------------------------------------------------------------------------- -Another difference is that while the Logstash Forwarder sends the hostname of -the server it is running on in the `host` field, Filebeat uses the -`beat.hostname` field for the same purpose. Because `host` is commonly used in -the Logstash plugin ecosystem, the Beats input plugin automatically copies -`beat.hostname` into `host`. +==== Filebeat uses "beat.hostname" for Sending the Hostname of the Server + +While the Logstash Forwarder sends the hostname of the server it's running on in +the `host` field, Filebeat uses the `beat.hostname` field for the same purpose. +Because `host` is commonly used in the Logstash plugin ecosystem, the Beats +input plugin automatically copies `beat.hostname` into `host`. -Similarly, the `file` field was renamed to `source`. If you rely on this field -being named `file` you can rename it by using the mutate filter in Logstash. For +==== The "file" Field Was Renamed to "source" + +The `file` field was renamed to `source`. If you rely on this field being +named `file`, you can rename it by using the mutate filter in Logstash. For example: [source,plain] @@ -397,8 +402,10 @@ filter { } ------------------------------------------------------------------------------------- -Finally, the `line` field was removed. This field wasn't correct in case of -restarts and making it correct would require a performance penalty. We recommend +==== The "line" Field Was Removed + +The `line` field was removed. This field wasn't correct after restarts, and +making it correct would have resulted in a performance penalty. We recommend using the `offset` field instead. === Other Changes diff --git a/docs/overview.asciidoc b/docs/overview.asciidoc index b33b7d041e76..1da978832013 100644 --- a/docs/overview.asciidoc +++ b/docs/overview.asciidoc @@ -2,7 +2,7 @@ Filebeat is a log data shipper initially based on the https://github.com/elastic/logstash-forwarder[Logstash-Forwarder] -source code. Installed as an agent on your servers, Filebeat monitors the log directories or specific log files, tails the files, +source code. Installed as an agent on your servers, Filebeat monitors the log directories or specific log files, tails the files, and forwards them either to https://www.elastic.co/products/logstash[Logstash] for parsing or directly to https://www.elastic.co/products/elasticsearch[Elasticsearch] for indexing. @@ -10,6 +10,5 @@ Here's how Filebeat works: When you start Filebeat, it starts one or more prospe image:./images/filebeat.png[Beats design] -Filebeat is a https://www.elastic.co/products/beats[Beat], and it is based on the libbeat framework. -The general configuration options for libbeat and how to setup Elasticsearch, Logstash, and Kibana are covered -https://www.elastic.co/guide/en/beats/libbeat/current/index.html[here]. +Filebeat is a https://www.elastic.co/products/beats[Beat], and it is based on the libbeat framework. +The general configuration options for libbeat and how to setup Elasticsearch, Logstash, and Kibana are covered in the {libbeat}/index.html[Beats Platform Reference].