From ac148136416b9ab0fa85ab5003d9f305d89279df Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 31 Jan 2024 15:07:08 +0200 Subject: [PATCH] Revert "Revert "x-pack/plugin/core: make automatic rollovers lazy" (#104734)" This reverts commit 807147d338dc8d86916b904353455b2801671daf. --- docs/changelog/104597.yaml | 6 +++++ .../IndexTemplateRegistryRolloverIT.java | 27 ++++++++++++++++--- .../core/template/IndexTemplateRegistry.java | 3 ++- .../template/IndexTemplateRegistryTests.java | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/104597.yaml diff --git a/docs/changelog/104597.yaml b/docs/changelog/104597.yaml new file mode 100644 index 000000000000..02fc78e1017a --- /dev/null +++ b/docs/changelog/104597.yaml @@ -0,0 +1,6 @@ +pr: 104597 +summary: "X-pack/plugin/core: make automatic rollovers lazy" +area: Data streams +type: enhancement +issues: + - 104083 diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java index 12f9ce685a8b..5834f0484df0 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java @@ -7,8 +7,11 @@ package org.elasticsearch.xpack.core.template; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -16,16 +19,21 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; import org.junit.Before; import java.util.Collection; import java.util.List; +import java.util.Locale; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; import static org.elasticsearch.xpack.core.template.RolloverEnabledTestTemplateRegistry.TEST_INDEX_PATTERN; import static org.elasticsearch.xpack.core.template.RolloverEnabledTestTemplateRegistry.TEST_INDEX_TEMPLATE_ID; import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) @@ -67,16 +75,29 @@ public void testRollover() throws Exception { assertNumberOfBackingIndices(1); registry.incrementVersion(); registry.clusterChanged(new ClusterChangedEvent(IndexTemplateRegistryRolloverIT.class.getName(), clusterService.state(), state)); + assertBusy(() -> assertTrue(getDataStream().rolloverOnWrite())); + assertNumberOfBackingIndices(1); + + String timestampValue = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + DocWriteResponse docWriteResponse = client().index( + new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, timestampValue), XContentType.JSON) + ).actionGet(); + assertThat(docWriteResponse.status().getStatus(), equalTo(201)); assertBusy(() -> assertNumberOfBackingIndices(2)); } private void assertNumberOfBackingIndices(final int expected) { + DataStream dataStream = getDataStream(); + assertThat(dataStream.getIndices(), hasSize(expected)); + assertThat(dataStream.getWriteIndex().getName(), endsWith(String.valueOf(expected))); + } + + private DataStream getDataStream() { GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { TEST_INDEX_PATTERN }); GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); List dataStreams = getDataStreamResponse.getDataStreams(); assertThat(dataStreams, hasSize(1)); - DataStream dataStream = dataStreams.get(0).getDataStream(); - assertThat(dataStream.getIndices(), hasSize(expected)); - assertThat(dataStream.getWriteIndex().getName(), endsWith(String.valueOf(expected))); + return dataStreams.get(0).getDataStream(); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index ecec5991ade3..db5746f5c1b4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -808,12 +808,13 @@ public void onFailure(Exception e) { ); for (String rolloverTarget : rolloverTargets) { logger.info( - "rolling over data stream [{}] as a followup to the upgrade of the [{}] index template [{}]", + "rolling over data stream [{}] lazily as a followup to the upgrade of the [{}] index template [{}]", rolloverTarget, getOrigin(), templateName ); RolloverRequest request = new RolloverRequest(rolloverTarget, null); + request.lazy(true); request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); executeAsyncWithOrigin( client.threadPool().getThreadContext(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java index 2b845affde63..b0127c000532 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java @@ -377,6 +377,7 @@ public void testAutomaticRollover() throws Exception { rolloverCounter.incrementAndGet(); RolloverRequest rolloverRequest = ((RolloverRequest) request); assertThat(rolloverRequest.getRolloverTarget(), startsWith("logs-my_app-")); + assertThat(rolloverRequest.isLazy(), equalTo(true)); } else if (action == TransportPutComposableIndexTemplateAction.TYPE) { putIndexTemplateCounter.incrementAndGet(); }