[ML] More advanced post-test cleanup of ML indices (#39049)

The .ml-annotations index is created asynchronously when
some other ML index exists.  This can interfere with the
post-test index deletion, as the .ml-annotations index
can be created after all other indices have been deleted.

This change adds an ML specific post-test cleanup step
that runs before the main cleanup and:

1. Checks if any ML indices exist
2. If so, waits for the .ml-annotations index to exist
3. Deletes the other ML indices found in step 1.
4. Calls the super class cleanup

This means that by the time the main post-test index
cleanup code runs:

1. The only ML index it has to delete will be the
   .ml-annotations index
2. No other ML indices will exist that could trigger
   recreation of the .ml-annotations index

Fixes #38952

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java

@@ -15,11 +15,14 @@
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
 /**
  * An extension to {@link ESSingleNodeTestCase} that adds node settings specifically needed for ML test cases.
  */
@@ -46,6 +49,33 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(LocalStateMachineLearning.class);
     }
 
+    /**
+     * This cleanup is to fix the problem described in
+     * https://github.com/elastic/elasticsearch/issues/38952
+     */
+    @Override
+    public void tearDown() throws Exception {
+        try {
+            logger.trace("[{}#{}]: ML-specific after test cleanup", getTestClass().getSimpleName(), getTestName());
+            String[] nonAnnotationMlIndices;
+            boolean mlAnnotationsIndexExists;
+            do {
+                String[] mlIndices = client().admin().indices().prepareGetIndex().addIndices(".ml-*").get().indices();
+                nonAnnotationMlIndices = Arrays.stream(mlIndices).filter(name -> name.startsWith(".ml-annotations") == false)
+                    .toArray(String[]::new);
+                mlAnnotationsIndexExists = mlIndices.length > nonAnnotationMlIndices.length;
+            } while (nonAnnotationMlIndices.length > 0 && mlAnnotationsIndexExists == false);
+            if (nonAnnotationMlIndices.length > 0) {
+                // Delete the ML indices apart from the annotations index.  The annotations index will be deleted by the
+                // base class cleanup.  We want to delete all the others first so that the annotations index doesn't get
+                // automatically recreated.
+                assertAcked(client().admin().indices().prepareDelete(nonAnnotationMlIndices).get());
+            }
+        } finally {
+            super.tearDown();
+        }
+    }
+
     protected void waitForMlTemplates() throws Exception {
         // block until the templates are installed
         assertBusy(() -> {