[ML] indicate overall deployment failure if all node routes are failed

docs/changelog/88378.yaml

@@ -0,0 +1,5 @@
+pr: 88378
+summary: Indicate overall deployment failure if all node routes are failed
+area: Machine Learning
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentState.java

@@ -13,7 +13,8 @@
 public enum AssignmentState {
     STARTING,
     STARTED,
-    STOPPING;
+    STOPPING,
+    FAILED; // Not persisted, calculated via route states
 
     public static AssignmentState fromString(String value) {
         return valueOf(value.toUpperCase(Locale.ROOT));

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java

@@ -519,7 +519,11 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalVInt(queueCapacity);
         out.writeInstant(startTime);
         out.writeList(nodeStats);
-        out.writeOptionalEnum(state);
+        if (AssignmentState.FAILED.equals(state) && out.getVersion().before(Version.V_8_4_0)) {
+            out.writeOptionalEnum(AssignmentState.STARTING);
+        } else {
+            out.writeOptionalEnum(state);
+        }
         out.writeOptionalString(reason);
         out.writeOptionalWriteable(allocationStatus);
     }

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
 import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.BertTokenizer;
@@ -347,6 +348,30 @@ public void testLiveDeploymentStats() throws IOException {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public void testFailedDeploymentStats() throws Exception {
+        String badModel = "bad_model";
+        String poorlyFormattedModelBase64 = "cG9vcmx5IGZvcm1hdHRlZCBtb2RlbAo=";
+        int length = Base64.getDecoder().decode(poorlyFormattedModelBase64).length;
+        createTrainedModel(badModel);
+        putVocabulary(List.of("once", "twice"), badModel);
+        Request request = new Request("PUT", "_ml/trained_models/" + badModel + "/definition/0");
+        request.setJsonEntity("""
+            {"total_definition_length":%s,"definition": "%s","total_parts": 1}""".formatted(length, poorlyFormattedModelBase64));
+        client().performRequest(request);
+        startDeployment(badModel, AllocationStatus.State.STARTING.toString());
+        assertBusy(() -> {
+            Response noInferenceCallsStatsResponse = getTrainedModelStats(badModel);
+            List<Map<String, Object>> stats = (List<Map<String, Object>>) entityAsMap(noInferenceCallsStatsResponse).get(
+                "trained_model_stats"
+            );
+            assertThat(stats, hasSize(1));
+
+            String assignmentState = (String) XContentMapValues.extractValue("deployment_stats.state", stats.get(0));
+            assertThat(assignmentState, equalTo(AssignmentState.FAILED.toString()));
+        });
+    }
+
     private void assertAtLeastOneOfTheseIsNotNull(String name, List<Map<String, Object>> nodes) {
         assertTrue("all nodes have null value for [" + name + "]", nodes.stream().anyMatch(n -> n.get(name) != null));
     }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java

@@ -143,6 +143,15 @@ protected void doExecute(
                 TrainedModelAssignment trainedModelAssignment = assignment.getModelAssignment(stats.getModelId());
                 if (trainedModelAssignment != null) {
                     stats.setState(trainedModelAssignment.getAssignmentState()).setReason(trainedModelAssignment.getReason().orElse(null));
+                    if (trainedModelAssignment.getNodeRoutingTable()
+                        .values()
+                        .stream()
+                        .allMatch(ri -> ri.getState().equals(RoutingState.FAILED))) {
+                        stats.setState(AssignmentState.FAILED);
+                        if (stats.getReason() == null) {
+                            stats.setReason("All node routes are failed; see node route reason for details");
+                        }
+                    }
                     if (trainedModelAssignment.getAssignmentState().isAnyOf(AssignmentState.STARTED, AssignmentState.STARTING)) {
                         stats.setAllocationStatus(trainedModelAssignment.calculateAllocationStatus().orElse(null));
                     }