From 6d943e9e53576700b8417164d0e4c899c5f84e52 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 17 Dec 2024 16:33:27 +0100
Subject: [PATCH] [ML] Ignore failures from renormalizing buckets in read-only
 index (#118674)

In anomaly detection, score renormalization will update the anomaly score in the result indices. However, if the index in the old format was marked as read-only, the score update will fail. Since this failure is expected, this PR suppresses the error logging in this specific case.
---
 docs/changelog/118674.yaml                    |  5 ++++
 .../JobRenormalizedResultsPersister.java      | 26 ++++++++++++++++++-
 2 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 docs/changelog/118674.yaml

diff --git a/docs/changelog/118674.yaml b/docs/changelog/118674.yaml
new file mode 100644
index 0000000000000..eeb90a3b38f66
--- /dev/null
+++ b/docs/changelog/118674.yaml
@@ -0,0 +1,5 @@
+pr: 118674
+summary: Ignore failures from renormalizing buckets in read-only index
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
index 3c0d2aca4deda..3c82841f1b99e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
@@ -8,10 +8,12 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -102,7 +104,29 @@ public void executeRequest() {
         try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
             BulkResponse addRecordsResponse = client.bulk(bulkRequest).actionGet();
             if (addRecordsResponse.hasFailures()) {
-                logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage());
+                // Implementation note: Ignore the failures from writing to the read-only index, as it comes
+                // from changing the index format version.
+                boolean hasNonReadOnlyFailures = false;
+                for (BulkItemResponse response : addRecordsResponse.getItems()) {
+                    if (response.isFailed() == false) {
+                        continue;
+                    }
+                    if (response.getFailureMessage().contains(IndexMetadata.INDEX_READ_ONLY_BLOCK.description())) {
+                        // We expect this to happen when the old index is made read-only and being reindexed
+                        logger.debug(
+                            "[{}] Ignoring failure to write renormalized results to a read-only index [{}]: {}",
+                            jobId,
+                            response.getFailure().getIndex(),
+                            response.getFailureMessage()
+                        );
+                    } else {
+                        hasNonReadOnlyFailures = true;
+                        break;
+                    }
+                }
+                if (hasNonReadOnlyFailures) {
+                    logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage());
+                }
             }
         }