[ML] Ignore failures from renormalizing buckets in read-only index

docs/changelog/118674.yaml

@@ -0,0 +1,5 @@
+pr: 118674
+summary: Ignore failures from renormalizing buckets in read-only index
+area: Machine Learning
+type: enhancement
+issues: []

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java

@@ -8,6 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
@@ -102,7 +103,24 @@ public void executeRequest() {
         try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
             BulkResponse addRecordsResponse = client.bulk(bulkRequest).actionGet();
             if (addRecordsResponse.hasFailures()) {
-                logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage());
+                // Implementation note: Ignore the failures from writing to the read-only index, as it comes
+                // from changing the index format version.
+                boolean hasNonReadOnlyFailures = false;
+                for (BulkItemResponse response : addRecordsResponse.getItems()) {
+                    if (response.isFailed() == false) {
+                        continue;
+                    }
+                    if (response.getFailureMessage().contains("index read-only")) {
+                        // We expect this to happen when the old index is made read-only and being reindexed
+                        logger.debug("[{}] Ignoring failure to write to read-only index: {}", jobId, response.getFailureMessage());
+                    } else {
+                        hasNonReadOnlyFailures = true;
+                        break;
+                    }
+                }
+                if (hasNonReadOnlyFailures) {
+                    logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage());
+                }
             }
         }