Fix and re-enable two monitoring migrate alerts test on 7.x branch

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -430,9 +430,20 @@ private void setupClusterAlertsTasks(ClusterState clusterState, boolean clusterS
                 if (watches != null && watches.allPrimaryShardsActive() == false) {
                     logger.trace("cannot manage cluster alerts because [.watches] index is not allocated");
                 } else if ((watches == null || indexExists) && watcherSetup.compareAndSet(false, true)) {
+                    logger.trace("installing monitoring watches");
                     getClusterAlertsInstallationAsyncActions(indexExists, asyncActions, pendingResponses);
+                } else {
+                    logger.trace("skipping installing monitoring watches, watches=[{}], indexExists=[{}], watcherSetup=[{}]",
+                        watches, indexExists, watcherSetup.get());
                 }
+            } else {
+                logger.trace("watches shouldn't be setup, because state=[{}] and clusterStateChange=[{}]", state.get(), clusterStateChange);
             }
+        } else {
+            logger.trace("watches can't be used, because xpack.watcher.enabled=[{}] and " +
+                    "xpack.monitoring.exporters._local.cluster_alerts.management.enabled=[{}]",
+                XPackSettings.WATCHER_ENABLED.get(config.settings()),
+                CLUSTER_ALERTS_MANAGEMENT_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings()));
         }
     }
 
@@ -581,6 +592,7 @@ private void getClusterAlertsInstallationAsyncActions(final boolean indexExists,
                                                                new ResponseActionListener<>("watch", uniqueWatchId, pendingResponses)));
                 }
             } else if (addWatch) {
+                logger.trace("adding monitoring watch [{}]", uniqueWatchId);
                 asyncActions.add(() -> putWatch(watcher, watchId, uniqueWatchId, pendingResponses));
             }
         }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java

@@ -31,11 +31,16 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.http.MockRequest;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
+import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequest;
+import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkResponse;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsAction;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsRequest;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsResponse;
@@ -49,6 +54,7 @@
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 import org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter;
 import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter;
+import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporterIntegTests;
 import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
 import org.junit.After;
 import org.junit.Before;
@@ -124,7 +130,9 @@ private void stopMonitoring() {
         ));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/66586")
+    @TestLogging(
+        value = "org.elasticsearch.xpack.monitoring.exporter.local:trace",
+        reason = "to ensure we log local exporter on trace level")
     public void testLocalAlertsRemoval() throws Exception {
         try {
             // start monitoring service
@@ -159,6 +167,9 @@ public void testLocalAlertsRemoval() throws Exception {
         }
     }
 
+    @TestLogging(
+        value = "org.elasticsearch.xpack.monitoring.exporter.local:trace",
+        reason = "to ensure we log local exporter on trace level")
     public void testRepeatedLocalAlertsRemoval() throws Exception {
         try {
             // start monitoring service
@@ -474,6 +485,18 @@ public void testRemoteAlertsRemoteDisallowsWatcher() throws Exception {
     }
 
     private void ensureInitialLocalResources() throws Exception {
+        // Should trigger setting up alert watches via LocalExporter#openBulk(...) and
+        // then eventually to LocalExporter#setupIfElectedMaster(...)
+        // Sometimes this last method doesn't install watches, because elected master node doesn't export monitor documents.
+        // and then these assertions here fail.
+        {
+            MonitoringBulkRequest request = new MonitoringBulkRequest();
+            request.add(LocalExporterIntegTests.createMonitoringBulkDoc());
+            String masterNode = internalCluster().getMasterName();
+            MonitoringBulkResponse response = client(masterNode).execute(MonitoringBulkAction.INSTANCE, request).actionGet();
+            assertThat(response.status(), equalTo(RestStatus.OK));
+        }
+
         waitForWatcherIndices();
         assertBusy(() -> {
             assertThat(indexExists(".monitoring-*"), is(true));

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java

@@ -303,7 +303,7 @@ private void checkMonitoringDocs() {
         }
     }
 
-    private static MonitoringBulkDoc createMonitoringBulkDoc() throws IOException {
+    public static MonitoringBulkDoc createMonitoringBulkDoc() throws IOException {
         final MonitoredSystem system = randomFrom(BEATS, KIBANA, LOGSTASH);
         final XContentType xContentType = randomFrom(XContentType.values());
         final BytesReference source;