From a03b1ae8736d8d12269de0eef23908d3799166cf Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:48:17 +0100 Subject: [PATCH 1/7] Ignore failures from writing to read-only index due to index format version change --- .../JobRenormalizedResultsPersister.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java index 3c0d2aca4deda..5d28a61efc072 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; @@ -102,7 +103,24 @@ public void executeRequest() { try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) { BulkResponse addRecordsResponse = client.bulk(bulkRequest).actionGet(); if (addRecordsResponse.hasFailures()) { - logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage()); + // Implementation note: Ignore the failures from writing to the read-only index, as it comes + // from changing the index format version. + boolean hasNonReadOnlyFailures = false; + for (BulkItemResponse response : addRecordsResponse.getItems()) { + if (response.isFailed() == false) { + continue; + } + if (response.getFailureMessage().contains("index read-only")) { + // We expect this to happen when the old index is made read-only and being reindexed + logger.debug("[{}] Ignoring failure to write to read-only index: {}", jobId, response.getFailureMessage()); + } else { + hasNonReadOnlyFailures = true; + break; + } + } + if (hasNonReadOnlyFailures) { + logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage()); + } } } From 1f18f8dfdcec293a03824fc285ebe93d2e348b54 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:50:18 +0100 Subject: [PATCH 2/7] Update docs/changelog/118674.yaml --- docs/changelog/118674.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/118674.yaml diff --git a/docs/changelog/118674.yaml b/docs/changelog/118674.yaml new file mode 100644 index 0000000000000..eeb90a3b38f66 --- /dev/null +++ b/docs/changelog/118674.yaml @@ -0,0 +1,5 @@ +pr: 118674 +summary: Ignore failures from renormalizing buckets in read-only index +area: Machine Learning +type: enhancement +issues: [] From aad98b0617b5693554ad5a57abafc7dbe7da56c0 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:28:32 +0100 Subject: [PATCH 3/7] Update x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java Co-authored-by: David Kyle --- .../ml/job/persistence/JobRenormalizedResultsPersister.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java index 5d28a61efc072..e9c2244ad96c1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -112,7 +112,7 @@ public void executeRequest() { } if (response.getFailureMessage().contains("index read-only")) { // We expect this to happen when the old index is made read-only and being reindexed - logger.debug("[{}] Ignoring failure to write to read-only index: {}", jobId, response.getFailureMessage()); + logger.debug("[{}] Ignoring failure to write renormalized results read-only index [{}]: {}", jobId, response.getFailure().getIndex(), response.getFailureMessage()); } else { hasNonReadOnlyFailures = true; break; From cc7bff76a56f9ec1fb3ac5295896a2b3de64a79e Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:28:39 +0100 Subject: [PATCH 4/7] Update x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java Co-authored-by: David Kyle --- .../ml/job/persistence/JobRenormalizedResultsPersister.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java index e9c2244ad96c1..67c503cfc8d1b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -110,7 +110,7 @@ public void executeRequest() { if (response.isFailed() == false) { continue; } - if (response.getFailureMessage().contains("index read-only")) { + if (response.getFailureMessage().contains(IndexMetadata.INDEX_READ_ONLY_BLOCK.desription())) { // We expect this to happen when the old index is made read-only and being reindexed logger.debug("[{}] Ignoring failure to write renormalized results read-only index [{}]: {}", jobId, response.getFailure().getIndex(), response.getFailureMessage()); } else { From 7b1f7818f8cd3e401cd2054cf93a3639458e6fea Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 17 Dec 2024 13:34:41 +0000 Subject: [PATCH 5/7] [CI] Auto commit changes from spotless --- .../job/persistence/JobRenormalizedResultsPersister.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java index 67c503cfc8d1b..aa1a522f73990 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -112,7 +112,12 @@ public void executeRequest() { } if (response.getFailureMessage().contains(IndexMetadata.INDEX_READ_ONLY_BLOCK.desription())) { // We expect this to happen when the old index is made read-only and being reindexed - logger.debug("[{}] Ignoring failure to write renormalized results read-only index [{}]: {}", jobId, response.getFailure().getIndex(), response.getFailureMessage()); + logger.debug( + "[{}] Ignoring failure to write renormalized results read-only index [{}]: {}", + jobId, + response.getFailure().getIndex(), + response.getFailureMessage() + ); } else { hasNonReadOnlyFailures = true; break; From ef0638c39f962d51bf7c8b2e7f3864035fa4f9c3 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:47:02 +0100 Subject: [PATCH 6/7] fix compilation issue --- .../ml/job/persistence/JobRenormalizedResultsPersister.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java index aa1a522f73990..6fb1f5835bb91 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -110,7 +111,7 @@ public void executeRequest() { if (response.isFailed() == false) { continue; } - if (response.getFailureMessage().contains(IndexMetadata.INDEX_READ_ONLY_BLOCK.desription())) { + if (response.getFailureMessage().contains(IndexMetadata.INDEX_READ_ONLY_BLOCK.description())) { // We expect this to happen when the old index is made read-only and being reindexed logger.debug( "[{}] Ignoring failure to write renormalized results read-only index [{}]: {}", From 80c0ded27ea3b9c1366eff640a92a76ac60899b2 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:32:22 +0100 Subject: [PATCH 7/7] Update x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java Co-authored-by: David Kyle --- .../ml/job/persistence/JobRenormalizedResultsPersister.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java index 6fb1f5835bb91..3c82841f1b99e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -114,7 +114,7 @@ public void executeRequest() { if (response.getFailureMessage().contains(IndexMetadata.INDEX_READ_ONLY_BLOCK.description())) { // We expect this to happen when the old index is made read-only and being reindexed logger.debug( - "[{}] Ignoring failure to write renormalized results read-only index [{}]: {}", + "[{}] Ignoring failure to write renormalized results to a read-only index [{}]: {}", jobId, response.getFailure().getIndex(), response.getFailureMessage()