Fix registry unsupported pipeline update

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java

@@ -185,6 +185,10 @@ public void clusterChanged(ClusterChangedEvent event) {
             return;
         }
 
+        if (isClusterReady(event) == false) {
+            return;
+        }
+
         // if this node is newer than the master node, we probably need to add the template, which might be newer than the
         // template the master node has, so we need potentially add new templates despite being not the master node
         DiscoveryNode localNode = event.state().getNodes().getLocalNode();
@@ -197,6 +201,14 @@ public void clusterChanged(ClusterChangedEvent event) {
         }
     }
 
+    /**
+     * A method that can be overridden to add additional conditions to be satisfied
+     * before installing the template registry components.
+     */
+    protected boolean isClusterReady(ClusterChangedEvent event) {
+        return true;
+    }
+
     /**
      * Whether the registry should only apply changes when running on the master node.
      * This is useful for plugins where certain actions are performed on master nodes

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java

@@ -6,7 +6,9 @@
  */
 package org.elasticsearch.xpack.application.analytics;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -35,8 +37,11 @@
 
 public class AnalyticsTemplateRegistry extends IndexTemplateRegistry {
 
+    // This registry requires all nodes to be at least 8.8.0
+    static final Version MIN_NODE_VERSION = Version.V_8_8_0;
+
     // This number must be incremented when we make changes to built-in templates.
-    static final int REGISTRY_VERSION = 1;
+    static final int REGISTRY_VERSION = 2;
 
     // ILM Policies configuration
     static final String EVENT_DATA_STREAM_ILM_POLICY_NAME = EVENT_DATA_STREAM_INDEX_PREFIX + "default_policy";
@@ -146,4 +151,11 @@ protected boolean requiresMasterNode() {
         // there and the ActionNotFoundTransportException errors are then prevented.
         return true;
     }
+
+    @Override
+    protected boolean isClusterReady(ClusterChangedEvent event) {
+        // Ensure templates are installed only once all nodes are updated to 8.8.0.
+        Version minNodeVersion = event.state().nodes().getMinNodeVersion();
+        return minNodeVersion.onOrAfter(MIN_NODE_VERSION);
+    }
 }

x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistryTests.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.application.analytics;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -405,6 +406,25 @@ public void testThatNonExistingPipelinesAreAddedImmediately() throws Exception {
         assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getIngestPipelines().size())));
     }
 
+    public void testThatNothingIsInstalledWhenAllNodesAreNotUpdated() {
+        DiscoveryNode updatedNode = TestDiscoveryNode.create("updatedNode");
+        DiscoveryNode outdatedNode = TestDiscoveryNode.create("outdatedNode", ESTestCase.buildNewFakeTransportAddress(), Version.V_8_7_0);
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .localNodeId("updatedNode")
+            .masterNodeId("updatedNode")
+            .add(updatedNode)
+            .add(outdatedNode)
+            .build();
+
+        client.setVerifier((a, r, l) -> {
+            fail("if some cluster mode are not updated to at least v.8.8.0 nothing should happen");
+            return null;
+        });
+
+        ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes);
+        registry.clusterChanged(event);
+    }
+
     // -------------
 
     /**

x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java

@@ -9,7 +9,9 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -36,6 +38,9 @@
 public class StackTemplateRegistry extends IndexTemplateRegistry {
     private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class);
 
+    // Current version of the registry requires all nodes to be at least 8.9.0.
+    public static final Version MIN_NODE_VERSION = Version.V_8_9_0;
+
     // The stack template registry version. This number must be incremented when we make changes
     // to built-in templates.
     public static final int REGISTRY_VERSION = 3;
@@ -253,4 +258,13 @@ protected boolean requiresMasterNode() {
         // there and the ActionNotFoundTransportException errors are then prevented.
         return true;
     }
+
+    @Override
+    protected boolean isClusterReady(ClusterChangedEvent event) {
+        // Ensure current version of the components are installed only once all nodes are updated to 8.9.0.
+        // This is necessary to prevent an error caused nby the usage of the ignore_missing_pipeline property
+        // in the pipeline processor, which has been introduced only in 8.9.0
+        Version minNodeVersion = event.state().nodes().getMinNodeVersion();
+        return minNodeVersion.onOrAfter(MIN_NODE_VERSION);
+    }
 }

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.stack;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -444,6 +445,25 @@ public void testThatMissingMasterNodeDoesNothing() {
         registry.clusterChanged(event);
     }
 
+    public void testThatNothingIsInstalledWhenAllNodesAreNotUpdated() {
+        DiscoveryNode updatedNode = TestDiscoveryNode.create("updatedNode");
+        DiscoveryNode outdatedNode = TestDiscoveryNode.create("outdatedNode", ESTestCase.buildNewFakeTransportAddress(), Version.V_8_8_0);
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .localNodeId("updatedNode")
+            .masterNodeId("updatedNode")
+            .add(updatedNode)
+            .add(outdatedNode)
+            .build();
+
+        client.setVerifier((a, r, l) -> {
+            fail("if some cluster mode are not updated to at least v.8.9.0 nothing should happen");
+            return null;
+        });
+
+        ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
+        registry.clusterChanged(event);
+    }
+
     // -------------
 
     /**

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java

@@ -112,21 +112,10 @@ public void testTrainedModelDeployment() throws Exception {
                     request.addParameter("wait_for_status", "yellow");
                     request.addParameter("timeout", "70s");
                 }));
-
-                // Workaround for an upgrade test failure where an ingest
-                // pipeline config cannot be parsed by older nodes:
-                // https://github.com/elastic/elasticsearch/issues/95766
-                //
-                // In version 8.3.1 ml stopped parsing the full ingest
-                // pipeline configuration so will avoid this problem.
-                // TODO remove this check once https://github.com/elastic/elasticsearch/issues/95766
-                // is resolved
-                if (UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_3_1)) {
-                    waitForDeploymentStarted(modelId);
-                    // attempt inference on new and old nodes multiple times
-                    for (int i = 0; i < 10; i++) {
-                        assertInfer(modelId);
-                    }
+                waitForDeploymentStarted(modelId);
+                // attempt inference on new and old nodes multiple times
+                for (int i = 0; i < 10; i++) {
+                    assertInfer(modelId);
                 }
             }
             case UPGRADED -> {

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlTrainedModelsUpgradeIT.java

@@ -74,19 +74,8 @@ public void testTrainedModelInference() throws Exception {
                     request.addParameter("timeout", "70s");
                 }));
                 List<String> modelIds = getTrainedModels();
-
-                // Workaround for an upgrade test failure where an ingest
-                // pipeline config cannot be parsed by older nodes:
-                // https://github.com/elastic/elasticsearch/issues/95766
-                //
-                // In version 8.3.1 ml stopped parsing the full ingest
-                // pipeline configuration so will avoid this problem.
-                // TODO remove this check once https://github.com/elastic/elasticsearch/issues/95766
-                // is resolved
-                if (UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_3_1)) {
-                    // Test that stats are serializable and can be gathered
-                    getTrainedModelStats();
-                }
+                // Test that stats are serializable and can be gathered
+                getTrainedModelStats();
                 // Verify that the pipelines still work and inference is possible
                 testInfer(modelIds);
             }