diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index eed4c7636ba04..2deff44fca4f0 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1154,7 +1154,7 @@ public void testIndexDocsWithCustomRoutingAllowed() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(false, true, null) + new ComposableIndexTemplate.DataStreamTemplate(false, true) ); client().execute( PutComposableIndexTemplateAction.INSTANCE, @@ -1804,7 +1804,7 @@ public void testPartitionedTemplate() throws IOException { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(false, true, null) + new ComposableIndexTemplate.DataStreamTemplate(false, true) ); ComposableIndexTemplate finalTemplate = template; client().execute( @@ -1830,7 +1830,7 @@ public void testPartitionedTemplate() throws IOException { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(false, true, null) + new ComposableIndexTemplate.DataStreamTemplate(false, true) ); client().execute( PutComposableIndexTemplateAction.INSTANCE, @@ -1856,7 +1856,7 @@ public void testPartitionedTemplate() throws IOException { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(false, false, null) + new ComposableIndexTemplate.DataStreamTemplate(false, false) ); ComposableIndexTemplate finalTemplate1 = template; Exception e = expectThrows( @@ -1898,7 +1898,7 @@ public void testSearchWithRouting() throws IOException, ExecutionException, Inte null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(false, true, null) + new ComposableIndexTemplate.DataStreamTemplate(false, true) ); client().execute( PutComposableIndexTemplateAction.INSTANCE, diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index cfd88293c1a4c..4fe2f81171b5b 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -10,15 +10,16 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -87,7 +88,7 @@ public void testTimeRanges() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES), + new ComposableIndexTemplate.DataStreamTemplate(false, false), null ) ); @@ -166,6 +167,122 @@ public void testTimeRanges() throws Exception { } } + public void testTsdbTemplatesWithComponentTemplate() throws Exception { + var mappingTemplate = """ + { + "_doc":{ + "properties": { + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""; + var putComponentTemplateRequest = new PutComponentTemplateAction.Request("id"); + putComponentTemplateRequest.componentTemplate( + new ComponentTemplate( + new Template( + Settings.builder().put("index.routing_path", "metricset").build(), + new CompressedXContent(mappingTemplate), + null + ), + null, + null + ) + ); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> client().execute(PutComponentTemplateAction.INSTANCE, putComponentTemplateRequest).actionGet() + ); + // We can't know whether the index template that uses this component template has data streams enabled: + // (The generation of certain settings index.mode, index.time_series_start_time etc. only doesn't happen) + assertThat(e.getMessage(), equalTo("[index.routing_path] requires [index.mode=time_series]")); + } + + public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exception { + var mappingTemplate = """ + { + "_doc":{ + "properties": { + "metricset": { + "type": "keyword" + } + } + } + }"""; + var request = new PutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + new ComposableIndexTemplate( + List.of("k8s*"), + new Template( + Settings.builder().put("index.routing_path", "metricset").build(), + new CompressedXContent(mappingTemplate), + null + ), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ) + ); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet() + ); + assertThat( + e.getCause().getCause().getMessage(), + equalTo( + "All fields that match routing_path must be keywords with [time_series_dimension: true] and " + + "without the [script] parameter. [metricset] was not [time_series_dimension: true]." + ) + ); + } + + public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception { + var mappingTemplate = """ + { + "_doc":{ + "properties": { + "metricset": { + "type": "long", + "time_series_dimension": true + } + } + } + }"""; + var request = new PutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + new ComposableIndexTemplate( + List.of("k8s*"), + new Template( + Settings.builder().put("index.routing_path", "metricset").build(), + new CompressedXContent(mappingTemplate), + null + ), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ) + ); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet() + ); + assertThat( + e.getCause().getCause().getMessage(), + equalTo( + "All fields that match routing_path must be keywords with [time_series_dimension: true] and " + + "without the [script] parameter. [metricset] was [long]." + ) + ); + } + static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index b717a9c36bd32..ddfc2a7fa3846 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -86,7 +86,6 @@ public class TsdbDataStreamRestIT extends ESRestTestCase { } }, "data_stream": { - "index_mode": "time_series" } }"""; @@ -463,7 +462,7 @@ public void testChangeTemplateIndexMode() throws Exception { e.getMessage(), containsString( "composable template [1] with index patterns [k8s*], priority [null]," - + " index_mode [null] would cause tsdb data streams [k8s] to no longer match a data stream template" + + " index.routing_path [] would cause tsdb data streams [k8s] to no longer match a data stream template" + " with a time_series index_mode" ) ); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 4fcf032ec74da..f24e26c7b97cc 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -30,7 +30,7 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider { public Settings getAdditionalIndexSettings( String indexName, String dataStreamName, - IndexMode templateIndexMode, + boolean timeSeries, Metadata metadata, Instant resolvedAt, Settings allSettings @@ -42,14 +42,16 @@ public Settings getAdditionalIndexSettings( // so checking that index_mode==null|standard and templateIndexMode == TIME_SERIES boolean migrating = dataStream != null && (dataStream.getIndexMode() == null || dataStream.getIndexMode() == IndexMode.STANDARD) - && templateIndexMode == IndexMode.TIME_SERIES; + && timeSeries; IndexMode indexMode; if (migrating) { indexMode = IndexMode.TIME_SERIES; } else if (dataStream != null) { indexMode = dataStream.getIndexMode(); + } else if (timeSeries) { + indexMode = IndexMode.TIME_SERIES; } else { - indexMode = templateIndexMode; + indexMode = null; } if (indexMode != null) { Settings.Builder builder = Settings.builder(); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index 5a85e21895db0..fb9c3acd21ff9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -287,7 +286,7 @@ public void cleanup() { private ClusterState createInitialState() { ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of("logs-*")) .template(new Template(Settings.builder().put("index.routing_path", "uid").build(), null, null)) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build(); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 9ea160d8b3d50..f50f4e3544298 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -39,7 +39,7 @@ public void testGetAdditionalIndexSettings() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + true, metadata, now, settings @@ -61,7 +61,7 @@ public void testGetAdditionalIndexSettingsLookAheadTime() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + true, metadata, now, settings @@ -89,7 +89,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() { var result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + true, metadata, now, settings @@ -139,7 +139,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi () -> provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + true, metadata, now, settings @@ -155,7 +155,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi ); } - public void testGetAdditionalIndexSettingsIndexModeNotSpecified() { + public void testGetAdditionalIndexSettingsNonTsdbTemplate() { Metadata metadata = Metadata.EMPTY_METADATA; String dataStreamName = "logs-app1"; @@ -164,7 +164,7 @@ public void testGetAdditionalIndexSettingsIndexModeNotSpecified() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - null, + false, metadata, Instant.ofEpochMilli(1L), settings @@ -172,22 +172,4 @@ public void testGetAdditionalIndexSettingsIndexModeNotSpecified() { assertThat(result.size(), equalTo(0)); } - public void testGetAdditionalIndexSettingsIndexModeStandardSpecified() { - Metadata metadata = Metadata.EMPTY_METADATA; - String dataStreamName = "logs-app1"; - - Settings settings = Settings.EMPTY; - var provider = new DataStreamIndexSettingsProvider(); - Settings result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 1), - dataStreamName, - IndexMode.STANDARD, - metadata, - Instant.ofEpochMilli(1L), - settings - ); - assertThat(result.size(), equalTo(1)); - assertThat(result.get(IndexSettings.MODE.getKey()), equalTo(IndexMode.STANDARD.name().toLowerCase(Locale.ROOT))); - } - } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java index e05cf9f8c7654..1ed71289a2f6e 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java @@ -47,7 +47,6 @@ protected Collection> getPlugins() { return List.of(DataStreamsPlugin.class); } - private String timestampFieldName = "@timestamp"; private final Set createdDataStreams = new HashSet<>(); @Override @@ -231,8 +230,8 @@ private String createDataStream() throws Exception { private String createDataStream(boolean hidden) throws Exception { String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault()); Template idxTemplate = new Template(null, new CompressedXContent(""" - {"properties":{"%s":{"type":"date"},"data":{"type":"keyword"}}} - """.formatted(timestampFieldName)), null); + {"properties":{"@timestamp":{"type":"date"},"data":{"type":"keyword"}}} + """), null); ComposableIndexTemplate template = new ComposableIndexTemplate( List.of(dataStreamName + "*"), idxTemplate, @@ -240,7 +239,7 @@ private String createDataStream(boolean hidden) throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(hidden, false, null), + new ComposableIndexTemplate.DataStreamTemplate(hidden, false), null ); assertTrue( @@ -265,7 +264,7 @@ private long createDocument(String dataStreamName) throws Exception { .source( JsonXContent.contentBuilder() .startObject() - .field(timestampFieldName, timestamp) + .field("@timestamp", timestamp) .field("data", randomAlphaOfLength(25)) .endObject() ) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java index 12a73ea07ad54..0eca0410ab15e 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java @@ -64,7 +64,7 @@ public void testRolloverClusterStateForDataStream() throws Exception { ); ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStream.getName() + "*")) .template(new Template(Settings.builder().put("index.routing_path", "uid").build(), null, null)) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build(); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); @@ -162,7 +162,7 @@ public void testRolloverAndMigrateDataStream() throws Exception { ); ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStream.getName() + "*")) .template(new Template(Settings.builder().put("index.routing_path", "uid").build(), null, null)) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build(); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); @@ -246,7 +246,7 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting ); ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStream.getName() + "*")) .template(new Template(Settings.builder().put("index.routing_path", "uid").build(), null, null)) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, randomBoolean() ? IndexMode.STANDARD : null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build(); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java new file mode 100644 index 0000000000000..b5d3378cd9a15 --- /dev/null +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.indices.EmptySystemIndices; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.ShardLimitValidator; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateTsdbMapping; +import static org.elasticsearch.common.settings.Settings.builder; +import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE; +import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Variant of MetadataIndexTemplateServiceTests in server module, but with {@link DataStreamIndexSettingsProvider}. + */ +public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { + + public void testValidateTsdbDataStreamsReferringTsdbTemplate() throws Exception { + var state = ClusterState.EMPTY_STATE; + final var service = getMetadataIndexTemplateService(); + var template = new ComposableIndexTemplate( + Collections.singletonList("logs-*-*"), + new Template(builder().put("index.routing_path", "uid").build(), new CompressedXContent(generateTsdbMapping()), null), + null, + 100L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ); + state = service.addIndexTemplateV2(state, false, "logs", template); + + var now = Instant.now(); + var mBuilder = Metadata.builder(state.getMetadata()); + DataStreamTestHelper.getClusterStateWithDataStream( + mBuilder, + "unreferenced", + List.of(Tuple.tuple(now.minus(2, ChronoUnit.HOURS), now)) + ); + DataStreamTestHelper.getClusterStateWithDataStream( + mBuilder, + "logs-mysql-default", + List.of(Tuple.tuple(now.minus(2, ChronoUnit.HOURS), now)) + ); + var stateWithDS = ClusterState.builder(state).metadata(mBuilder).build(); + + var e = expectThrows(IllegalArgumentException.class, () -> { + ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate( + Collections.singletonList("logs-*-*"), + null, + null, + 100L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(false, false), + null + ); + service.addIndexTemplateV2(stateWithDS, false, "logs", nonDSTemplate); + }); + + assertThat( + e.getMessage(), + containsString( + "would cause tsdb data streams [logs-mysql-default] to no longer match a data stream template with a time_series index_mode" + ) + ); + } + + private MetadataIndexTemplateService getMetadataIndexTemplateService() { + var indicesService = getInstanceFromNode(IndicesService.class); + var clusterService = getInstanceFromNode(ClusterService.class); + var indexSettingProviders = new IndexSettingProviders(Set.of(new DataStreamIndexSettingsProvider())); + var createIndexService = new MetadataCreateIndexService( + Settings.EMPTY, + clusterService, + indicesService, + null, + createTestShardLimitService(randomIntBetween(1, 1000)), + new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null), + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + null, + xContentRegistry(), + EmptySystemIndices.INSTANCE, + true, + indexSettingProviders + ); + return new MetadataIndexTemplateService( + clusterService, + createIndexService, + indicesService, + new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), + xContentRegistry(), + EmptySystemIndices.INSTANCE, + indexSettingProviders + ); + } + + public static ShardLimitValidator createTestShardLimitService(int maxShardsPerNode) { + // Use a mocked clusterService - for unit tests we won't be updating the setting anyway. + ClusterService clusterService = mock(ClusterService.class); + Settings limitOnlySettings = Settings.builder().put(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey(), maxShardsPerNode).build(); + when(clusterService.getClusterSettings()).thenReturn( + new ClusterSettings(limitOnlySettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + + return new ShardLimitValidator(limitOnlySettings, clusterService); + } + +} diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 9f867ab871055..3de76adc2a451 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -11,11 +11,11 @@ setup: name: my-template1 body: index_patterns: [k8s*] - data_stream: - index_mode : time_series + data_stream: {} template: settings: index: + mode: time_series number_of_replicas: 0 number_of_shards: 2 routing_path: [metricset, time_series_dimension] diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 62959f67c38f7..9629b87b98ad6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -242,11 +242,12 @@ private RolloverResult rolloverDataStream( ); } + final Metadata metadata = currentState.getMetadata(); final ComposableIndexTemplate templateV2; final SystemDataStreamDescriptor systemDataStreamDescriptor; if (dataStream.isSystem() == false) { systemDataStreamDescriptor = null; - templateV2 = lookupTemplateForDataStream(dataStreamName, currentState.metadata()); + templateV2 = lookupTemplateForDataStream(dataStreamName, metadata); } else { systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName); if (systemDataStreamDescriptor == null) { @@ -278,7 +279,7 @@ private RolloverResult rolloverDataStream( createIndexClusterStateRequest, silent, (builder, indexMetadata) -> builder.put( - ds.rollover(indexMetadata.getIndex(), newGeneration, templateV2.getDataStreamTemplate().getIndexMode()) + ds.rollover(indexMetadata.getIndex(), newGeneration, metadata.isTimeSeriesTemplate(templateV2)) ) ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 13750ca94ca7f..9ffbdec2422af 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -207,6 +207,7 @@ public static Template resolveTemplate( final SystemIndices systemIndices, Set indexSettingProviders ) throws Exception { + var metadata = simulatedState.getMetadata(); Settings templateSettings = resolveSettings(simulatedState.metadata(), matchingTemplate); List> resolvedAliases = MetadataIndexTemplateService.resolveAliases( @@ -229,7 +230,7 @@ public static Template resolveTemplate( Settings result = provider.getAdditionalIndexSettings( indexName, template.getDataStreamTemplate() != null ? indexName : null, - template.getDataStreamTemplate() != null ? template.getDataStreamTemplate().getIndexMode() : null, + template.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(template), simulatedState.getMetadata(), now, templateSettings diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 75f4c76293b44..2a8ea2dc1b6bd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -17,8 +17,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -307,42 +305,28 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { private static final ParseField HIDDEN = new ParseField("hidden"); private static final ParseField ALLOW_CUSTOM_ROUTING = new ParseField("allow_custom_routing"); - private static final ParseField INDEX_MODE = new ParseField("index_mode"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "data_stream_template", false, - args -> { - IndexMode indexMode; - if (IndexSettings.isTimeSeriesModeEnabled()) { - indexMode = args[2] != null ? IndexMode.fromString((String) args[2]) : null; - } else { - indexMode = null; - } - return new DataStreamTemplate(args[0] != null && (boolean) args[0], args[1] != null && (boolean) args[1], indexMode); - } + args -> new DataStreamTemplate(args[0] != null && (boolean) args[0], args[1] != null && (boolean) args[1]) ); static { PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_CUSTOM_ROUTING); - if (IndexSettings.isTimeSeriesModeEnabled()) { - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), INDEX_MODE); - } } private final boolean hidden; private final boolean allowCustomRouting; - private final IndexMode indexMode; public DataStreamTemplate() { - this(false, false, null); + this(false, false); } - public DataStreamTemplate(boolean hidden, boolean allowCustomRouting, IndexMode indexMode) { + public DataStreamTemplate(boolean hidden, boolean allowCustomRouting) { this.hidden = hidden; this.allowCustomRouting = allowCustomRouting; - this.indexMode = indexMode; } DataStreamTemplate(StreamInput in) throws IOException { @@ -352,10 +336,13 @@ public DataStreamTemplate(boolean hidden, boolean allowCustomRouting, IndexMode } else { allowCustomRouting = false; } - if (in.getVersion().onOrAfter(Version.V_8_1_0)) { - indexMode = in.readOptionalEnum(IndexMode.class); - } else { - indexMode = null; + if (in.getVersion().onOrAfter(Version.V_8_1_0) && in.getVersion().before(Version.V_8_3_0)) { + // Accidentally included index_mode to binary node to node protocol in previous releases. + // (index_mode is removed and was part of code based when tsdb was behind a feature flag) + // (index_mode was behind a feature in the xcontent parser, so it could never actually used) + // (this used to be an optional enum, so just need to (de-)serialize a false boolean value here) + boolean value = in.readBoolean(); + assert value == false : "expected false, because this used to be an optional enum that never got set"; } } @@ -389,19 +376,15 @@ public boolean isAllowCustomRouting() { return allowCustomRouting; } - @Nullable - public IndexMode getIndexMode() { - return indexMode; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(hidden); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeBoolean(allowCustomRouting); } - if (out.getVersion().onOrAfter(Version.V_8_1_0)) { - out.writeOptionalEnum(indexMode); + if (out.getVersion().onOrAfter(Version.V_8_1_0) && out.getVersion().before(Version.V_8_3_0)) { + // See comment in constructor. + out.writeBoolean(false); } } @@ -410,9 +393,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field("hidden", hidden); builder.field(ALLOW_CUSTOM_ROUTING.getPreferredName(), allowCustomRouting); - if (indexMode != null) { - builder.field(INDEX_MODE.getPreferredName(), indexMode); - } builder.endObject(); return builder; } @@ -422,12 +402,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; DataStreamTemplate that = (DataStreamTemplate) o; - return hidden == that.hidden && allowCustomRouting == that.allowCustomRouting && indexMode == that.indexMode; + return hidden == that.hidden && allowCustomRouting == that.allowCustomRouting; } @Override public int hashCode() { - return Objects.hash(hidden, allowCustomRouting, indexMode); + return Objects.hash(hidden, allowCustomRouting); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 1f60433379c40..da3829ec1bdb2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -268,26 +268,26 @@ public IndexMode getIndexMode() { * Performs a rollover on a {@code DataStream} instance and returns a new instance containing * the updated list of backing indices and incremented generation. * - * @param writeIndex new write index - * @param generation new generation - * @param indexModeFromTemplate the index mode as is defined in the template that created this data stream + * @param writeIndex new write index + * @param generation new generation + * @param timeSeries whether the template that created this data stream is in time series mode * * @return new {@code DataStream} instance with the rollover operation applied */ - public DataStream rollover(Index writeIndex, long generation, IndexMode indexModeFromTemplate) { + public DataStream rollover(Index writeIndex, long generation, boolean timeSeries) { ensureNotReplicated(); - return unsafeRollover(writeIndex, generation, indexModeFromTemplate); + return unsafeRollover(writeIndex, generation, timeSeries); } /** - * Like {@link #rollover(Index, long, IndexMode)}, but does no validation, use with care only. + * Like {@link #rollover(Index, long, boolean)}, but does no validation, use with care only. */ - public DataStream unsafeRollover(Index writeIndex, long generation, IndexMode indexModeFromTemplate) { + public DataStream unsafeRollover(Index writeIndex, long generation, boolean timeSeries) { IndexMode indexMode = this.indexMode; // This allows for migrating a data stream to be a tsdb data stream: // (only if index_mode=null|standard then allow it to be set to time_series) - if ((indexMode == null || indexMode == IndexMode.STANDARD) && indexModeFromTemplate == IndexMode.TIME_SERIES) { + if ((indexMode == null || indexMode == IndexMode.STANDARD) && timeSeries) { indexMode = IndexMode.TIME_SERIES; } @@ -298,7 +298,7 @@ public DataStream unsafeRollover(Index writeIndex, long generation, IndexMode in /** * Performs a dummy rollover on a {@code DataStream} instance and returns the tuple of the next write index name and next generation - * that this {@code DataStream} should roll over to using {@link #rollover(Index, long, IndexMode)}. + * that this {@code DataStream} should roll over to using {@link #rollover(Index, long, boolean)}. * * @param clusterMetadata Cluster metadata * diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index bcddacf699671..192333cffab7a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -896,6 +896,25 @@ public Map templatesV2() { .orElse(Collections.emptyMap()); } + public boolean isTimeSeriesTemplate(ComposableIndexTemplate indexTemplate) { + var template = indexTemplate.template(); + if (indexTemplate.getDataStreamTemplate() == null || template == null) { + return false; + } + + var settings = MetadataIndexTemplateService.resolveSettings(indexTemplate, componentTemplates()); + if (IndexMetadata.INDEX_ROUTING_PATH.exists(settings)) { + // No need to validate the index.mode=time_series validation takes care of validating that the mentioned fields exist in the + // mapping and that it refers to fields of type keyword and with time_series_dimension enabled. + return true; + } + + // in a followup change: check the existence of keyword fields of type keyword and time_series_dimension attribute enabled in + // the template. In this case the index.routing_path setting can be generated from the mapping. + + return false; + } + public Map dataStreams() { return Optional.ofNullable((DataStreamMetadata) this.custom(DataStreamMetadata.TYPE)) .map(DataStreamMetadata::dataStreams) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index c8aeaf468250f..d8104141ef1fa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -203,6 +203,7 @@ static ClusterState createDataStream( ); } + final var metadata = currentState.metadata(); final boolean isSystem = systemDataStreamDescriptor != null; final ComposableIndexTemplate template = isSystem ? systemDataStreamDescriptor.getComposableIndexTemplate() @@ -244,14 +245,12 @@ static ClusterState createDataStream( assert writeIndex != null; assert writeIndex.mapping() != null : "no mapping found for backing index [" + writeIndex.getIndex().getName() + "]"; - String fieldName = ComposableIndexTemplate.DataStreamTemplate.getTimestampField(); - DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName); List dsBackingIndices = backingIndices.stream() .map(IndexMetadata::getIndex) .collect(Collectors.toCollection(ArrayList::new)); dsBackingIndices.add(writeIndex.getIndex()); boolean hidden = isSystem || template.getDataStreamTemplate().isHidden(); - final IndexMode indexMode = template.getDataStreamTemplate().getIndexMode(); + final IndexMode indexMode = metadata.isTimeSeriesTemplate(template) ? IndexMode.TIME_SERIES : null; DataStream newDataStream = new DataStream( dataStreamName, dsBackingIndices, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index a3902e2d87dac..13323b598388e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -839,6 +839,7 @@ static Settings aggregateIndexSettings( Set indexSettingProviders ) { final boolean isDataStreamIndex = request.dataStreamName() != null; + final var metadata = currentState.getMetadata(); // Create builders for the template and request settings. We transform these into builders // because we may want settings to be "removed" from these prior to being set on the new @@ -851,11 +852,10 @@ static Settings aggregateIndexSettings( final Settings.Builder additionalIndexSettings = Settings.builder(); final Settings templateAndRequestSettings = Settings.builder().put(combinedTemplateSettings).put(request.settings()).build(); - final IndexMode matchingIndexMode = Optional.of(request) + final boolean timeSeriesTemplate = Optional.of(request) .map(CreateIndexClusterStateUpdateRequest::matchingTemplate) - .map(ComposableIndexTemplate::getDataStreamTemplate) - .map(ComposableIndexTemplate.DataStreamTemplate::getIndexMode) - .orElse(null); + .map(metadata::isTimeSeriesTemplate) + .orElse(false); // Loop through all the explicit index setting providers, adding them to the // additionalIndexSettings map @@ -865,7 +865,7 @@ static Settings aggregateIndexSettings( provider.getAdditionalIndexSettings( request.index(), request.dataStreamName(), - matchingIndexMode, + timeSeriesTemplate, currentState.getMetadata(), resolvedAt, templateAndRequestSettings diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 6c20357083292..17ff8f7dd1f2e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -630,6 +630,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT var finalTemplate = Optional.ofNullable(indexTemplate.template()); var finalSettings = Settings.builder(); final var now = Instant.now(); + final var metadata = currentState.getMetadata(); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { @@ -637,7 +638,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT provider.getAdditionalIndexSettings( "validate-index-name", indexTemplate.getDataStreamTemplate() != null ? "validate-data-stream-name" : null, - indexTemplate.getDataStreamTemplate() != null ? indexTemplate.getDataStreamTemplate().getIndexMode() : null, + indexTemplate.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(indexTemplate), currentState.getMetadata(), now, finalTemplate.map(Template::settings).orElse(Settings.EMPTY) @@ -743,6 +744,7 @@ private static void validateTsdbDataStreamsReferringTsdbTemplate( String templateName, ComposableIndexTemplate newTemplate ) { + Metadata currentMetadata = state.getMetadata(); Metadata updatedMetadata = null; Set dataStreamsWithNonTsdbTemplate = null; @@ -756,7 +758,7 @@ private static void validateTsdbDataStreamsReferringTsdbTemplate( } var matchingTemplate = findV2Template(updatedMetadata, dataStream.getName(), false); if (templateName.equals(matchingTemplate)) { - if (newTemplate.getDataStreamTemplate().getIndexMode() != IndexMode.TIME_SERIES) { + if (currentMetadata.isTimeSeriesTemplate(newTemplate) == false) { if (dataStreamsWithNonTsdbTemplate == null) { dataStreamsWithNonTsdbTemplate = new HashSet<>(); } @@ -766,6 +768,8 @@ private static void validateTsdbDataStreamsReferringTsdbTemplate( } if (dataStreamsWithNonTsdbTemplate != null) { + var settings = MetadataIndexTemplateService.resolveSettings(newTemplate, currentMetadata.componentTemplates()); + var routingPaths = IndexMetadata.INDEX_ROUTING_PATH.get(settings); throw new IllegalArgumentException( "composable template [" + templateName @@ -774,9 +778,9 @@ private static void validateTsdbDataStreamsReferringTsdbTemplate( + ", priority [" + newTemplate.priority() + "]" - + ", index_mode [" - + newTemplate.getDataStreamTemplate().getIndexMode() - + "] " + + ", index.routing_path " + + routingPaths + + " " + "would cause tsdb data streams " + dataStreamsWithNonTsdbTemplate + " to no longer match a data stream template with a time_series index_mode" diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java index 582372f04d724..54839eaad9d77 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; @@ -236,7 +235,7 @@ public static class DefaultHotAllocationSettingProvider implements IndexSettingP public Settings getAdditionalIndexSettings( String indexName, String dataStreamName, - IndexMode templateIndexMode, + boolean timeSeries, Metadata metadata, Instant resolvedAt, Settings allSettings diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java index c79c9a9adcf72..aa972cdf5a011 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java @@ -21,10 +21,9 @@ public interface IndexSettingProvider { /** * Returns explicitly set default index {@link Settings} for the given index. This should not * return null. - * * @param indexName The name of the new index being created * @param dataStreamName The name of the data stream if the index being created is part of a data stream otherwise null - * @param templateIndexMode The index mode from the data stream template of the matching template. + * @param timeSeries Whether the template is in time series mode. * @param metadata The current metadata instance that doesn't yet contain the index to be created * @param resolvedAt The time the request to create this new index was accepted. * @param allSettings All the setting resolved from the template that matches and any setting defined on the create index request @@ -32,7 +31,7 @@ public interface IndexSettingProvider { Settings getAdditionalIndexSettings( String indexName, String dataStreamName, - IndexMode templateIndexMode, + boolean timeSeries, Metadata metadata, Instant resolvedAt, Settings allSettings diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java index 38b68df728e81..a3a8818505b76 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java @@ -34,7 +34,7 @@ protected DataStreamTemplate createTestInstance() { public static DataStreamTemplate randomInstance() { IndexMode indexMode = randomBoolean() ? randomFrom(IndexMode.values()) : null; - return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean(), indexMode); + return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean()); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index d59cb6c86c224..aad6735daef78 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -60,7 +60,7 @@ protected DataStream createTestInstance() { public void testRollover() { DataStream ds = DataStreamTestHelper.randomInstance().promoteDataStream(); Tuple newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA); - final DataStream rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), null); + final DataStream rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), false); assertThat(rolledDs.getName(), equalTo(ds.getName())); assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); @@ -86,7 +86,7 @@ public void testRolloverWithConflictingBackingIndexName() { } final Tuple newCoordinates = ds.nextWriteIndexAndGeneration(builder.build()); - final DataStream rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), null); + final DataStream rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), false); assertThat(rolledDs.getName(), equalTo(ds.getName())); assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + numConflictingIndices + 1)); @@ -113,7 +113,7 @@ public void testRolloverIndexMode() { ); var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA); - var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TIME_SERIES); + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), true); assertThat(rolledDs.getName(), equalTo(ds.getName())); assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); @@ -138,8 +138,7 @@ public void testRolloverIndexMode_keepIndexMode() { ); var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA); - IndexMode indexMode = randomBoolean() ? IndexMode.STANDARD : null; - var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), indexMode); + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), false); assertThat(rolledDs.getName(), equalTo(ds.getName())); assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 146d26f85ba2a..2a947e4d3feef 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -21,10 +21,8 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; @@ -39,8 +37,6 @@ import org.elasticsearch.xcontent.XContentParseException; import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -2005,57 +2001,6 @@ public void testDataStreamsUsingTemplates() throws Exception { MetadataIndexTemplateService.innerRemoveIndexTemplateV2(stateWithTwoTemplates, "logs"); } - public void testValidateTsdbDataStreamsReferringTsdbTemplate() throws Exception { - ClusterState state = ClusterState.EMPTY_STATE; - final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); - ComposableIndexTemplate template = new ComposableIndexTemplate( - Collections.singletonList("logs-*-*"), - null, - null, - 100L, - null, - null, - new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES), - null - ); - state = service.addIndexTemplateV2(state, false, "logs", template); - - Instant now = Instant.now(); - Metadata.Builder mBuilder = new Metadata.Builder(state.getMetadata()); - DataStreamTestHelper.getClusterStateWithDataStream( - mBuilder, - "unreferenced", - List.of(Tuple.tuple(now.minus(2, ChronoUnit.HOURS), now)) - ); - DataStreamTestHelper.getClusterStateWithDataStream( - mBuilder, - "logs-mysql-default", - List.of(Tuple.tuple(now.minus(2, ChronoUnit.HOURS), now)) - ); - ClusterState stateWithDS = ClusterState.builder(state).metadata(mBuilder).build(); - - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { - ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate( - Collections.singletonList("logs-*-*"), - null, - null, - 100L, - null, - null, - new ComposableIndexTemplate.DataStreamTemplate(false, false, randomBoolean() ? null : IndexMode.STANDARD), - null - ); - service.addIndexTemplateV2(stateWithDS, false, "logs", nonDSTemplate); - }); - - assertThat( - e.getMessage(), - containsString( - "would cause tsdb data streams [logs-mysql-default] to no longer match a data stream template with a time_series index_mode" - ) - ); - } - private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { ThreadPool testThreadPool = mock(ThreadPool.class); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index ecf80715b0d48..5ee531ecdbd76 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -161,6 +161,23 @@ public static String generateMapping(String timestampFieldName) { }""".formatted(timestampFieldName); } + public static String generateTsdbMapping() { + return """ + { + "_doc":{ + "properties": { + "@timestamp": { + "type": "date" + }, + "uid": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""; + } + public static String generateMapping(String timestampFieldName, String type) { return "{\n" + " \"" diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index 1989fece08243..88065c858f200 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -558,7 +558,7 @@ private SingleForecast forecast(Metadata metadata, IndexAbstraction.DataStream s for (int i = 0; i < numberNewIndices; ++i) { final String uuid = UUIDs.randomBase64UUID(); final Tuple rolledDataStreamInfo = dataStream.unsafeNextWriteIndexAndGeneration(state.metadata()); - dataStream = dataStream.unsafeRollover(new Index(rolledDataStreamInfo.v1(), uuid), rolledDataStreamInfo.v2(), null); + dataStream = dataStream.unsafeRollover(new Index(rolledDataStreamInfo.v1(), uuid), rolledDataStreamInfo.v2(), false); // this unintentionally copies the in-sync allocation ids too. This has the fortunate effect of these indices // not being regarded new by the disk threshold decider, thereby respecting the low watermark threshold even for primaries. diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java index 7d3f6e1c47440..fd6f75e39e764 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java @@ -136,7 +136,7 @@ public void testCalculateUsage() { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(false, false, null) + new ComposableIndexTemplate.DataStreamTemplate(false, false) ) ) ) @@ -205,7 +205,7 @@ public void testCalculateUsage() { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(false, false, null) + new ComposableIndexTemplate.DataStreamTemplate(false, false) ) ) )