diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index cfedcdfc4e38..7691de5eb300 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -194,6 +194,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Experimental feature setup.template.append_fields added. {pull}6024[6024] - Add appender support to autodiscover {pull}6469[6469] - Add add_host_metadata processor {pull}5968[5968] +- Retry configuration to load dashboards if Kibana is not reachable when the beat starts. {pull}6560[6560] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index a38b6ef046f6..beb10bff54a4 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -744,6 +744,17 @@ output.elasticsearch: # how to install the dashboards by first querying Elasticsearch. #setup.dashboards.always_kibana: false +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 5744480e5d23..97f28caea7c5 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1223,6 +1223,17 @@ output.elasticsearch: # how to install the dashboards by first querying Elasticsearch. #setup.dashboards.always_kibana: false +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index a41c2c77a943..1fcedeae1e1e 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -853,6 +853,17 @@ output.elasticsearch: # how to install the dashboards by first querying Elasticsearch. #setup.dashboards.always_kibana: false +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index b1a896857dd7..7eeefa2c261d 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -639,6 +639,17 @@ output.elasticsearch: # how to install the dashboards by first querying Elasticsearch. #setup.dashboards.always_kibana: false +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 41fda773fdba..f453be9544ab 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -1,6 +1,7 @@ package instance import ( + "context" cryptRand "crypto/rand" "encoding/json" "flag" @@ -294,9 +295,10 @@ func (b *Beat) launch(bt beat.Creator) error { return beat.GracefulExit } - svc.HandleSignals(beater.Stop) + ctx, cancel := context.WithCancel(context.Background()) + svc.HandleSignals(beater.Stop, cancel) - err = b.loadDashboards(false) + err = b.loadDashboards(ctx, false) if err != nil { return err } @@ -382,7 +384,8 @@ func (b *Beat) Setup(bt beat.Creator, template, dashboards, machineLearning bool } if dashboards { - err = b.loadDashboards(true) + fmt.Println("Loading dashboards (Kibana must be running and reachable)") + err = b.loadDashboards(context.Background(), true) if err != nil { return err } @@ -565,7 +568,7 @@ func openRegular(filename string) (*os.File, error) { return f, nil } -func (b *Beat) loadDashboards(force bool) error { +func (b *Beat) loadDashboards(ctx context.Context, force bool) error { if setup || force { // -setup implies dashboards.enabled=true if b.Config.Dashboards == nil { @@ -583,7 +586,7 @@ func (b *Beat) loadDashboards(force bool) error { if b.Config.Output.Name() == "elasticsearch" { esConfig = b.Config.Output.Config() } - err := dashboards.ImportDashboards(b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""), + err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""), b.Config.Kibana, esConfig, b.Config.Dashboards, nil) if err != nil { return fmt.Errorf("Error importing Kibana dashboards: %v", err) diff --git a/libbeat/dashboards/config.go b/libbeat/dashboards/config.go index 5c6db62c15b9..ce7d09b1e85f 100644 --- a/libbeat/dashboards/config.go +++ b/libbeat/dashboards/config.go @@ -1,5 +1,7 @@ package dashboards +import "time" + type Config struct { Enabled bool `config:"enabled"` KibanaIndex string `config:"kibana_index"` @@ -11,11 +13,23 @@ type Config struct { OnlyDashboards bool `config:"only_dashboards"` OnlyIndex bool `config:"only_index"` AlwaysKibana bool `config:"always_kibana"` + Retry *Retry `config:"retry"` +} + +type Retry struct { + Enabled bool `config:"enabled"` + Interval time.Duration `config:"interval"` + Maximum uint `config:"maximum"` } var defaultConfig = Config{ KibanaIndex: ".kibana", AlwaysKibana: false, + Retry: &Retry{ + Enabled: false, + Interval: time.Second, + Maximum: 0, + }, } var ( defaultDirectory = "kibana" diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index d5556c50a5bb..b22761ddd8b9 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -1,6 +1,7 @@ package dashboards import ( + "context" "errors" "fmt" "path/filepath" @@ -24,6 +25,7 @@ const ( // via the kibana dashboard loader plugin. For older versions of the Elastic Stack // we write the dashboards directly into the .kibana index. func ImportDashboards( + ctx context.Context, beatName, hostname, homePath string, kibanaConfig, esConfig, dashboardsConfig *common.Config, msgOutputter MessageOutputter, @@ -102,16 +104,16 @@ func ImportDashboards( case importViaES: return ImportDashboardsViaElasticsearch(esLoader) case importViaKibana: - return setupAndImportDashboardsViaKibana(hostname, kibanaConfig, &dashConfig, msgOutputter) + return setupAndImportDashboardsViaKibana(ctx, hostname, kibanaConfig, &dashConfig, msgOutputter) default: return errors.New("Elasticsearch or Kibana configuration missing for loading dashboards.") } } -func setupAndImportDashboardsViaKibana(hostname string, kibanaConfig *common.Config, +func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kibanaConfig *common.Config, dashboardsConfig *Config, msgOutputter MessageOutputter) error { - kibanaLoader, err := NewKibanaLoader(kibanaConfig, dashboardsConfig, hostname, msgOutputter) + kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter) if err != nil { return fmt.Errorf("fail to create the Kibana loader: %v", err) } diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 065a0a8bbd8f..d4922b9488b8 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -1,10 +1,12 @@ package dashboards import ( + "context" "encoding/json" "fmt" "io/ioutil" "net/url" + "time" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -21,13 +23,13 @@ type KibanaLoader struct { msgOutputter MessageOutputter } -func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) { +func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) { if cfg == nil || !cfg.Enabled() { return nil, fmt.Errorf("Kibana is not configured or enabled") } - client, err := kibana.NewKibanaClient(cfg) + client, err := getKibanaClient(ctx, cfg, dashboardsConfig.Retry, 0) if err != nil { return nil, fmt.Errorf("Error creating Kibana client: %v", err) } @@ -45,6 +47,22 @@ func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, hostname stri return &loader, nil } +func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, retryAttempt uint) (*kibana.Client, error) { + client, err := kibana.NewKibanaClient(cfg) + if err != nil { + if retryCfg.Enabled && (retryCfg.Maximum == 0 || retryCfg.Maximum > retryAttempt) { + select { + case <-ctx.Done(): + return nil, err + case <-time.After(retryCfg.Interval): + return getKibanaClient(ctx, cfg, retryCfg, retryAttempt+1) + } + } + return nil, fmt.Errorf("Error creating Kibana client: %v", err) + } + return client, nil +} + func (loader KibanaLoader) ImportIndex(file string) error { params := url.Values{} params.Set("force", "true") //overwrite the existing dashboards diff --git a/libbeat/docs/dashboardsconfig.asciidoc b/libbeat/docs/dashboardsconfig.asciidoc index dd5e255d22fc..bd7f9ab163a8 100644 --- a/libbeat/docs/dashboardsconfig.asciidoc +++ b/libbeat/docs/dashboardsconfig.asciidoc @@ -99,3 +99,20 @@ NOTE: This setting only works for Kibana 6.0 and newer. Force loading of dashboards using the Kibana API without querying Elasticsearch for the version The default is `false`. + +[float] +==== `setup.dashboards.retry.enabled` + +If this option is set to true, and Kibana is not reachable at the time when dashboards are loaded, + {beatname_uc} will retry to reconnect to Kibana instead of exiting with an error. Disabled by default. + +[float] +==== `setup.dashboards.retry.interval` + +Duration interval between Kibana connection retries. Defaults to 1 second. + +[float] +==== `setup.dashboards.retry.maximum` + +Maximum number of retries before exiting with an error. Set to 0 for unlimited retrying. +Default is unlimited. diff --git a/libbeat/service/service.go b/libbeat/service/service.go index 1fe39581f92d..b05d20c22ad4 100644 --- a/libbeat/service/service.go +++ b/libbeat/service/service.go @@ -1,6 +1,7 @@ package service import ( + "context" "expvar" "flag" "fmt" @@ -22,7 +23,7 @@ import ( // HandleSignals manages OS signals that ask the service/daemon to stop. // The stopFunction should break the loop in the Beat so that // the service shut downs gracefully. -func HandleSignals(stopFunction func()) { +func HandleSignals(stopFunction func(), cancel context.CancelFunc) { var callback sync.Once // On ^C or SIGTERM, gracefully stop the sniffer @@ -31,6 +32,7 @@ func HandleSignals(stopFunction func()) { go func() { <-sigc logp.Debug("service", "Received sigterm/sigint, stopping") + cancel() callback.Do(stopFunction) }() diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index e2fea98ae6e6..c2d081c90cb4 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1147,6 +1147,17 @@ output.elasticsearch: # how to install the dashboards by first querying Elasticsearch. #setup.dashboards.always_kibana: false +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index ee646179737e..349432834eba 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1107,6 +1107,17 @@ output.elasticsearch: # how to install the dashboards by first querying Elasticsearch. #setup.dashboards.always_kibana: false +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 4997044a8dae..6dff01e3bf89 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -668,6 +668,17 @@ output.elasticsearch: # how to install the dashboards by first querying Elasticsearch. #setup.dashboards.always_kibana: false +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch