diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 5c383bd4e1706..92b8ef1de4be5 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -430,9 +430,20 @@ private void setupClusterAlertsTasks(ClusterState clusterState, boolean clusterS if (watches != null && watches.allPrimaryShardsActive() == false) { logger.trace("cannot manage cluster alerts because [.watches] index is not allocated"); } else if ((watches == null || indexExists) && watcherSetup.compareAndSet(false, true)) { + logger.trace("installing monitoring watches"); getClusterAlertsInstallationAsyncActions(indexExists, asyncActions, pendingResponses); + } else { + logger.trace("skipping installing monitoring watches, watches=[{}], indexExists=[{}], watcherSetup=[{}]", + watches, indexExists, watcherSetup.get()); } + } else { + logger.trace("watches shouldn't be setup, because state=[{}] and clusterStateChange=[{}]", state.get(), clusterStateChange); } + } else { + logger.trace("watches can't be used, because xpack.watcher.enabled=[{}] and " + + "xpack.monitoring.exporters._local.cluster_alerts.management.enabled=[{}]", + XPackSettings.WATCHER_ENABLED.get(config.settings()), + CLUSTER_ALERTS_MANAGEMENT_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings())); } } @@ -581,6 +592,7 @@ private void getClusterAlertsInstallationAsyncActions(final boolean indexExists, new ResponseActionListener<>("watch", uniqueWatchId, pendingResponses))); } } else if (addWatch) { + logger.trace("adding monitoring watch [{}]", uniqueWatchId); asyncActions.add(() -> putWatch(watcher, watchId, uniqueWatchId, pendingResponses)); } } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java index e97c0f251de67..e60ca4a584ec3 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java @@ -31,11 +31,16 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.http.MockRequest; import org.elasticsearch.test.http.MockResponse; import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction; +import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequest; +import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkResponse; import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsAction; import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsRequest; import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsResponse; @@ -49,6 +54,7 @@ import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil; import org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter; import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter; +import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporterIntegTests; import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase; import org.junit.After; import org.junit.Before; @@ -124,7 +130,9 @@ private void stopMonitoring() { )); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/66586") + @TestLogging( + value = "org.elasticsearch.xpack.monitoring.exporter.local:trace", + reason = "to ensure we log local exporter on trace level") public void testLocalAlertsRemoval() throws Exception { try { // start monitoring service @@ -159,6 +167,9 @@ public void testLocalAlertsRemoval() throws Exception { } } + @TestLogging( + value = "org.elasticsearch.xpack.monitoring.exporter.local:trace", + reason = "to ensure we log local exporter on trace level") public void testRepeatedLocalAlertsRemoval() throws Exception { try { // start monitoring service @@ -474,6 +485,18 @@ public void testRemoteAlertsRemoteDisallowsWatcher() throws Exception { } private void ensureInitialLocalResources() throws Exception { + // Should trigger setting up alert watches via LocalExporter#openBulk(...) and + // then eventually to LocalExporter#setupIfElectedMaster(...) + // Sometimes this last method doesn't install watches, because elected master node doesn't export monitor documents. + // and then these assertions here fail. + { + MonitoringBulkRequest request = new MonitoringBulkRequest(); + request.add(LocalExporterIntegTests.createMonitoringBulkDoc()); + String masterNode = internalCluster().getMasterName(); + MonitoringBulkResponse response = client(masterNode).execute(MonitoringBulkAction.INSTANCE, request).actionGet(); + assertThat(response.status(), equalTo(RestStatus.OK)); + } + waitForWatcherIndices(); assertBusy(() -> { assertThat(indexExists(".monitoring-*"), is(true)); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java index 5aa6e729e72eb..d66f768f22151 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java @@ -303,7 +303,7 @@ private void checkMonitoringDocs() { } } - private static MonitoringBulkDoc createMonitoringBulkDoc() throws IOException { + public static MonitoringBulkDoc createMonitoringBulkDoc() throws IOException { final MonitoredSystem system = randomFrom(BEATS, KIBANA, LOGSTASH); final XContentType xContentType = randomFrom(XContentType.values()); final BytesReference source;