From a9b35f444c5fb5c7ea5ea6f33540122cf2a808c8 Mon Sep 17 00:00:00 2001 From: Maria Ralli Date: Tue, 21 Apr 2020 16:22:12 +0300 Subject: [PATCH] Delay warning about missing x-pack (#54265) Currently, when monitoring is enabled in a freshly-installed cluster, the non-master nodes log a warning message indicating that master may not have x-pack installed. The message is often printed even when the master does have x-pack installed but takes some time to setup the local exporter for monitoring. This commit adds the local exporter setting `wait_master.timeout` which defaults to 30 seconds. The setting configures the time that the non-master nodes should wait for master to setup monitoring. After the time elapses, they log a message to the user about possible missing x-pack installation on master. The logging of this warning was moved from `resolveBulk()` to `openBulk()` since `resolveBulk()` is called only on cluster updates and the message might not be logged until a new cluster update occurs. Closes #40898 Co-authored-by: Elastic Machine --- .../settings/monitoring-settings.asciidoc | 5 ++ .../xpack/monitoring/exporter/Exporters.java | 1 + .../exporter/local/LocalExporter.java | 49 +++++++++++++++---- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/docs/reference/settings/monitoring-settings.asciidoc b/docs/reference/settings/monitoring-settings.asciidoc index fab26f8facd8a..b7032c8987fc9 100644 --- a/docs/reference/settings/monitoring-settings.asciidoc +++ b/docs/reference/settings/monitoring-settings.asciidoc @@ -153,6 +153,11 @@ Whether to create cluster alerts for this cluster. The default value is `true`. To use this feature, {watcher} must be enabled. If you have a basic license, cluster alerts are not displayed. +`wait_master.timeout`:: + +(<>) Time to wait for the master node to setup `local` exporter for monitoring. +After that, the non-master nodes will warn the user for possible missing X-Pack configuration. Defaults to `30s`. + [float] [[http-exporter-settings]] ==== HTTP Exporter Settings diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java index a4b7e8edac184..aa10f4821e0ca 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java @@ -254,6 +254,7 @@ public static List> getSettings() { List> settings = new ArrayList<>(); settings.addAll(Exporter.getSettings()); settings.addAll(HttpExporter.getSettings()); + settings.addAll(LocalExporter.getSettings()); return settings; } diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 0f6e5d4176bcc..2b7a6aaa4cfb0 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -28,10 +28,13 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.ingest.PipelineConfiguration; @@ -86,6 +89,16 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle public static final String TYPE = "local"; + /** + * Time to wait for the master node to setup local exporter for monitoring. + * After that, the non-master nodes will warn the user for possible missing configuration. + */ + public static final Setting.AffixSetting WAIT_MASTER_TIMEOUT_SETTING = Setting.affixKeySetting( + "xpack.monitoring.exporters.", + "wait_master.timeout", + (key) -> Setting.timeSetting(key, TimeValue.timeValueSeconds(30), Property.Dynamic, Property.NodeScope) + ); + private final Client client; private final ClusterService clusterService; private final XPackLicenseState licenseState; @@ -96,8 +109,10 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle private final AtomicReference state = new AtomicReference<>(State.INITIALIZED); private final AtomicBoolean installingSomething = new AtomicBoolean(false); - private final AtomicBoolean waitedForSetup = new AtomicBoolean(false); private final AtomicBoolean watcherSetup = new AtomicBoolean(false); + private final AtomicBoolean stateInitialized = new AtomicBoolean(false); + + private long stateInitializedTime; public LocalExporter(Exporter.Config config, Client client, CleanerService cleanerService) { super(config); @@ -116,6 +131,13 @@ public LocalExporter(Exporter.Config config, Client client, CleanerService clean @Override public void clusterChanged(ClusterChangedEvent event) { + // Save the time right after the cluster state is initialized/recovered + // to use it later for LocalExporter#WAIT_MASTER_TIMEOUT_SETTING + if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK) == false) { + if (stateInitialized.getAndSet(true) == false) { + stateInitializedTime = client.threadPool().relativeTimeInMillis(); + } + } if (state.get() == State.INITIALIZED) { resolveBulk(event.state(), true); } @@ -144,6 +166,17 @@ boolean isExporterReady() { @Override public void openBulk(final ActionListener listener) { if (state.get() != State.RUNNING) { + // wait for some time before informing the user for possible missing x-pack configuration on master + final TimeValue masterTimeout = WAIT_MASTER_TIMEOUT_SETTING.getConcreteSettingForNamespace(config.name()) + .get(config.settings()); + TimeValue timeElapsed = TimeValue.timeValueMillis(client.threadPool().relativeTimeInMillis() - stateInitializedTime); + if (timeElapsed.compareTo(masterTimeout) > 0) { + logger.info( + "waiting for elected master node [{}] to setup local exporter [{}] (does it have x-pack installed?)", + clusterService.state().nodes().getMasterNode(), + config.name() + ); + } listener.onResponse(null); } else { try { @@ -179,14 +212,8 @@ LocalBulk resolveBulk(ClusterState clusterState, boolean clusterStateChange) { // elected master node needs to setup templates; non-master nodes need to wait for it to be setup if (clusterService.state().nodes().isLocalNodeElectedMaster()) { setup = setupIfElectedMaster(clusterState, templates, clusterStateChange); - } else if (setupIfNotElectedMaster(clusterState, templates.keySet()) == false) { - // the first pass will be false so that we don't bother users if the master took one-go to setup - if (waitedForSetup.getAndSet(true)) { - logger.info("waiting for elected master node [{}] to setup local exporter [{}] (does it have x-pack installed?)", - clusterService.state().nodes().getMasterNode(), config.name()); - } - - setup = false; + } else { + setup = setupIfNotElectedMaster(clusterState, templates.keySet()); } // any failure/delay to setup the local exporter stops it until the next pass (10s by default) @@ -651,4 +678,8 @@ public void onFailure(Exception e) { } + public static List> getSettings() { + return List.of(WAIT_MASTER_TIMEOUT_SETTING); + } + }