Merge remote-tracking branch 'origin/main' into write-field-index

felixbarny · Aug 19, 2023 · 8e5dbde · 8e5dbde
2 parents 9eb1ad2 + 001fcfb
commit 8e5dbde
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 26 deletions.
diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.downsample;
 
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteable;
@@ -235,4 +236,28 @@ public static Rounding.Prepared createRounding(final String expr, final String t
         rounding.timeZone(ZoneId.of(timeZone, ZoneId.SHORT_IDS));
         return rounding.build().prepareForUnknown();
     }
+
+    /**
+     * Generates a downsample index name in the format
+     * prefix-fixedInterval-baseIndexName
+     *
+     * Note that this looks for the base index name of the provided index metadata via the
+     * {@link IndexMetadata#INDEX_DOWNSAMPLE_SOURCE_NAME_KEY} setting. This means that in case
+     * the provided index was already downsampled, we'll use the original source index (of the
+     * current provided downsample index) as the base index name.
+     */
+    public static String generateDownsampleIndexName(
+        String prefix,
+        IndexMetadata sourceIndexMetadata,
+        DateHistogramInterval fixedInterval
+    ) {
+        String downsampleSourceName = sourceIndexMetadata.getSettings().get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY);
+        String sourceIndexName;
+        if (downsampleSourceName != null) {
+            sourceIndexName = downsampleSourceName;
+        } else {
+            sourceIndexName = sourceIndexMetadata.getIndex().getName();
+        }
+        return prefix + fixedInterval + "-" + sourceIndexName;
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/downsample/DonwsampleConfigTests.java b/server/src/test/java/org/elasticsearch/action/downsample/DonwsampleConfigTests.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.downsample;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.action.downsample.DownsampleConfig.generateDownsampleIndexName;
+import static org.hamcrest.Matchers.is;
+
+public class DonwsampleConfigTests extends ESTestCase {
+
+    public void testGenerateDownsampleIndexName() {
+        {
+            String indexName = "test";
+            IndexMetadata indexMeta = IndexMetadata.builder(indexName).settings(indexSettings(IndexVersion.current(), 1, 0)).build();
+            assertThat(generateDownsampleIndexName("downsample-", indexMeta, new DateHistogramInterval("1h")), is("downsample-1h-test"));
+        }
+
+        {
+            String downsampledIndex = "downsample-1h-test";
+            IndexMetadata indexMeta = IndexMetadata.builder(downsampledIndex)
+                .settings(indexSettings(IndexVersion.current(), 1, 0).put(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY, "test"))
+                .build();
+            assertThat(generateDownsampleIndexName("downsample-", indexMeta, new DateHistogramInterval("8h")), is("downsample-8h-test"));
+        }
+    }
+}
diff --git a/...e/src/main/java/org/elasticsearch/xpack/core/ilm/DownsamplePrepareLifeCycleStateStep.java b/...e/src/main/java/org/elasticsearch/xpack/core/ilm/DownsamplePrepareLifeCycleStateStep.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 
+import static org.elasticsearch.action.downsample.DownsampleConfig.generateDownsampleIndexName;
 import static org.elasticsearch.xpack.core.ilm.DownsampleAction.DOWNSAMPLED_INDEX_PREFIX;
 
 /**
@@ -45,7 +46,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
 
         LifecycleExecutionState.Builder newLifecycleState = LifecycleExecutionState.builder(lifecycleState);
-        final String downsampleIndexName = generateDownsampleIndexName(indexMetadata, fixedInterval);
+        final String downsampleIndexName = generateDownsampleIndexName(DOWNSAMPLED_INDEX_PREFIX, indexMetadata, fixedInterval);
         newLifecycleState.setDownsampleIndexName(downsampleIndexName);
 
         return LifecycleExecutionStateUtils.newClusterStateWithLifecycleState(
@@ -60,14 +61,4 @@ public boolean isRetryable() {
         return true;
     }
 
-    static String generateDownsampleIndexName(IndexMetadata sourceIndexMetadata, DateHistogramInterval fixedInterval) {
-        String downsampleSourceName = sourceIndexMetadata.getSettings().get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY);
-        String sourceIndexName;
-        if (downsampleSourceName != null) {
-            sourceIndexName = downsampleSourceName;
-        } else {
-            sourceIndexName = sourceIndexMetadata.getIndex().getName();
-        }
-        return DOWNSAMPLED_INDEX_PREFIX + sourceIndexName + "-" + fixedInterval;
-    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleStepTests.java
@@ -28,6 +28,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.elasticsearch.action.downsample.DownsampleConfig.generateDownsampleIndexName;
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
 import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.elasticsearch.common.IndexNameGenerator.generateValidIndexName;
@@ -96,7 +97,7 @@ private IndexMetadata getIndexMetadata(String index, String lifecycleName, Downs
         lifecycleState.setAction(step.getKey().action());
         lifecycleState.setStep(step.getKey().name());
         lifecycleState.setIndexCreationDate(randomNonNegativeLong());
-        lifecycleState.setDownsampleIndexName(DownsamplePrepareLifeCycleStateStep.generateDownsampleIndexName(im, step.getFixedInterval()));
+        lifecycleState.setDownsampleIndexName(generateDownsampleIndexName(DOWNSAMPLED_INDEX_PREFIX, im, step.getFixedInterval()));
         return IndexMetadata.builder(im).putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()).build();
     }
 
@@ -105,7 +106,7 @@ private static void assertDownsampleActionRequest(DownsampleAction.Request reque
         assertThat(request.getSourceIndex(), equalTo(sourceIndex));
         assertThat(
             request.getTargetIndex(),
-            equalTo(DOWNSAMPLED_INDEX_PREFIX + sourceIndex + "-" + request.getDownsampleConfig().getFixedInterval())
+            equalTo(DOWNSAMPLED_INDEX_PREFIX + request.getDownsampleConfig().getFixedInterval() + "-" + sourceIndex)
         );
     }
 
@@ -185,10 +186,7 @@ public void testPerformActionDownsampleInProgressIndexExists() {
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
-        String downsampleIndex = DownsamplePrepareLifeCycleStateStep.generateDownsampleIndexName(
-            sourceIndexMetadata,
-            step.getFixedInterval()
-        );
+        String downsampleIndex = generateDownsampleIndexName(DOWNSAMPLED_INDEX_PREFIX, sourceIndexMetadata, step.getFixedInterval());
         LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
         lifecycleState.setPhase(step.getKey().phase());
         lifecycleState.setAction(step.getKey().action());

diff --git a/...nternalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java b/...nternalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
@@ -191,7 +191,7 @@ public boolean validateClusterForming() {
                 }
             })).start();
 
-            final String targetIndex = "downsample-" + sourceIndex + "-1h";
+            final String targetIndex = "downsample-1h-" + sourceIndex;
             startDownsampleTaskViaIlm(sourceIndex, targetIndex, disruptionStart, disruptionEnd);
             waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty());
             ensureStableCluster(cluster.numDataAndMasterNodes());

diff --git a/...ti-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/...ti-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.xcontent.ObjectPath;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.core.ilm.CheckNotDataStreamWriteIndexStep;
@@ -350,9 +349,8 @@ public void testDownsampleTwice() throws Exception {
         String now = DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(Instant.now());
         index(client(), dataStream, true, null, "@timestamp", now, "volume", 11.0, "metricset", randomAlphaOfLength(5));
 
-        var getDataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + dataStream));
-        String firstBackingIndex = ObjectPath.eval("data_streams.0.indices.0.index_name", responseAsMap(getDataStreamResponse));
-        logger.info("firstBackingIndex: {}", firstBackingIndex);
+        String firstBackingIndex = DataStream.getDefaultBackingIndexName(dataStream, 1);
+        logger.info("--> firstBackingIndex: {}", firstBackingIndex);
         assertBusy(
             () -> assertThat(
                 "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore",
@@ -366,8 +364,8 @@ public void testDownsampleTwice() throws Exception {
         // Manual rollover the original index such that it's not the write index in the data stream anymore
         rolloverMaxOneDocCondition(client(), dataStream);
 
-        String downsampleIndexName = "downsample-" + firstBackingIndex + "-1m";
-        String downsampleOfDownsampleIndexName = "downsample-" + firstBackingIndex + "-1h";
+        String downsampleIndexName = "downsample-1m-" + firstBackingIndex;
+        String downsampleOfDownsampleIndexName = "downsample-1h-" + firstBackingIndex;
         try {
             assertBusy(() -> {
                 assertThat(indexExists(downsampleOfDownsampleIndexName), is(true));
@@ -416,9 +414,9 @@ public static String getRollupIndexName(RestClient client, String originalIndexN
         throws IOException {
         String endpoint = "/"
             + DownsampleAction.DOWNSAMPLED_INDEX_PREFIX
-            + originalIndexName
-            + "-"
             + fixedInterval
+            + "-"
+            + originalIndexName
             + "*/?expand_wildcards=all";
         Response response = client.performRequest(new Request("GET", endpoint));
         Map<String, Object> asMap = responseAsMap(response);