From 5c30d129a43bd67ce09e911d4708b906c2ac2a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 8 Nov 2022 18:45:36 +0100 Subject: [PATCH 01/10] Forecast normalized write load during rollovers --- .../rollover/MetadataRolloverService.java | 9 + .../cluster/metadata/IndexMetadata.java | 60 ++++++- .../common/settings/IndexScopedSettings.java | 4 + .../elasticsearch/index/IndexSettings.java | 27 +++ .../index/shard/IndexWriteLoad.java | 4 +- .../index/shard/IndexWriteLoadForecast.java | 162 ++++++++++++++++++ .../shard/IndexWriteLoadForecastTests.java | 147 ++++++++++++++++ x-pack/plugin/write-load-plugin/build.gradle | 21 +++ ...WriteLoadForecastIndexSettingProvider.java | 51 ++++++ .../xpack/writeload/WriteLoadPlugin.java | 36 ++++ 10 files changed, 513 insertions(+), 8 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoadForecast.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadForecastTests.java create mode 100644 x-pack/plugin/write-load-plugin/build.gradle create mode 100644 x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadForecastIndexSettingProvider.java create mode 100644 x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadPlugin.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 9ab67dd4672b3..d1aad0af2fa5d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -28,9 +28,11 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.IndexWriteLoad; +import org.elasticsearch.index.shard.IndexWriteLoadForecast; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.snapshots.SnapshotInProgressException; @@ -307,6 +309,13 @@ private RolloverResult rolloverDataStream( ) .build(); + newState = IndexWriteLoadForecast.maybeIncludeWriteLoadForecast( + dataStream, + newState, + TimeValue.timeValueDays(7), + TimeValue.timeValueHours(8) + ); + return new RolloverResult(newWriteIndexName, originalWriteIndex.getName(), newState); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 8461197149567..724954ac1050b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.IndexWriteLoad; +import org.elasticsearch.index.shard.IndexWriteLoadForecast; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.rest.RestStatus; @@ -69,6 +70,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.OptionalDouble; import java.util.Set; import java.util.function.Function; @@ -523,6 +525,8 @@ public Iterator> settings() { public static final String KEY_PRIMARY_TERMS = "primary_terms"; public static final String KEY_WRITE_LOAD = "write_load"; + public static final String KEY_WRITE_LOAD_FORECAST = "write_load_forecast"; + public static final String INDEX_STATE_FILE_PREFIX = "state-"; static final Version SYSTEM_INDEX_FLAG_ADDED = Version.V_7_10_0; @@ -608,6 +612,8 @@ public Iterator> settings() { private final Instant timeSeriesEnd; @Nullable private final IndexWriteLoad writeLoad; + @Nullable + private final IndexWriteLoadForecast writeLoadForecast; private IndexMetadata( final Index index, @@ -651,7 +657,8 @@ private IndexMetadata( @Nullable final Instant timeSeriesStart, @Nullable final Instant timeSeriesEnd, final Version indexCompatibilityVersion, - @Nullable final IndexWriteLoad writeLoad + @Nullable final IndexWriteLoad writeLoad, + @Nullable final IndexWriteLoadForecast writeLoadForecast ) { this.index = index; this.version = version; @@ -703,6 +710,7 @@ private IndexMetadata( this.timeSeriesStart = timeSeriesStart; this.timeSeriesEnd = timeSeriesEnd; this.writeLoad = writeLoad; + this.writeLoadForecast = writeLoadForecast; assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards; } @@ -752,7 +760,8 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { this.timeSeriesStart, this.timeSeriesEnd, this.indexCompatibilityVersion, - this.writeLoad + this.writeLoad, + this.writeLoadForecast ); } @@ -808,7 +817,8 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) this.timeSeriesStart, this.timeSeriesEnd, this.indexCompatibilityVersion, - this.writeLoad + this.writeLoad, + this.writeLoadForecast ); } @@ -862,7 +872,8 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) { this.timeSeriesStart, this.timeSeriesEnd, this.indexCompatibilityVersion, - this.writeLoad + this.writeLoad, + this.writeLoadForecast ); } @@ -916,7 +927,8 @@ public IndexMetadata withTimestampRange(IndexLongFieldRange timestampRange) { this.timeSeriesStart, this.timeSeriesEnd, this.indexCompatibilityVersion, - this.writeLoad + this.writeLoad, + this.writeLoadForecast ); } @@ -966,7 +978,8 @@ public IndexMetadata withIncrementedVersion() { this.timeSeriesStart, this.timeSeriesEnd, this.indexCompatibilityVersion, - this.writeLoad + this.writeLoad, + this.writeLoadForecast ); } @@ -1162,6 +1175,14 @@ public IndexWriteLoad getWriteLoad() { return writeLoad; } + public OptionalDouble getForecastedWriteLoadForShard(int shardId) { + if (IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING.exists(settings)) { + return OptionalDouble.of(IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING.get(settings)); + } + + return writeLoadForecast == null ? OptionalDouble.empty() : writeLoadForecast.getForecastedWriteLoadForShard(shardId); + } + public static final String INDEX_RESIZE_SOURCE_UUID_KEY = "index.resize.source.uuid"; public static final String INDEX_RESIZE_SOURCE_NAME_KEY = "index.resize.source.name"; public static final Setting INDEX_RESIZE_SOURCE_UUID = Setting.simpleString(INDEX_RESIZE_SOURCE_UUID_KEY); @@ -1397,6 +1418,7 @@ private static class IndexMetadataDiff implements Diff { private final boolean isSystem; private final IndexLongFieldRange timestampRange; private final IndexWriteLoad indexWriteLoad; + private final IndexWriteLoadForecast indexWriteLoadForecast; IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); @@ -1431,6 +1453,7 @@ private static class IndexMetadataDiff implements Diff { isSystem = after.isSystem; timestampRange = after.timestampRange; indexWriteLoad = after.writeLoad; + indexWriteLoadForecast = after.writeLoadForecast; } private static final DiffableUtils.DiffableValueReader ALIAS_METADATA_DIFF_VALUE_READER = @@ -1483,8 +1506,10 @@ private static class IndexMetadataDiff implements Diff { timestampRange = IndexLongFieldRange.readFrom(in); if (in.getVersion().onOrAfter(WRITE_LOAD_ADDED)) { indexWriteLoad = in.readOptionalWriteable(IndexWriteLoad::new); + indexWriteLoadForecast = in.readOptionalWriteable(IndexWriteLoadForecast::new); } else { indexWriteLoad = null; + indexWriteLoadForecast = null; } } @@ -1518,6 +1543,7 @@ public void writeTo(StreamOutput out) throws IOException { timestampRange.writeTo(out); if (out.getVersion().onOrAfter(WRITE_LOAD_ADDED)) { out.writeOptionalWriteable(indexWriteLoad); + out.writeOptionalWriteable(indexWriteLoadForecast); } } @@ -1546,6 +1572,7 @@ public IndexMetadata apply(IndexMetadata part) { builder.system(isSystem); builder.timestampRange(timestampRange); builder.indexWriteLoad(indexWriteLoad); + builder.indexWriteLoadForecast(indexWriteLoadForecast); return builder.build(); } } @@ -1610,6 +1637,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function> settings() { Property.IndexSettingDeprecatedInV7AndRemovedInV8 ); + public static final Setting FORECAST_WRITE_LOAD_SETTING = Setting.boolSetting( + "index.forecast_write_load", + false, + Property.Dynamic, + Property.IndexScope, + Property.PrivateIndex + ); + + public static final Setting DEFAULT_WRITE_LOAD_SETTING = Setting.doubleSetting( + "index.default_write_load", + 0.0, + 0.0, + 1.0, + Property.Dynamic, + Property.IndexScope + ); + + public static final Setting DEFAULT_INTERNAL_WRITE_LOAD_SETTING = Setting.doubleSetting( + "index.internal_default_write_load", + 0.0, + 0.0, + 1.0, + Property.Dynamic, + Property.IndexScope, + Property.PrivateIndex + ); + private final Index index; private final Version version; private final Logger logger; diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoad.java b/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoad.java index e2922e0539685..d65151bc42538 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoad.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoad.java @@ -184,13 +184,15 @@ private Builder(int numShards) { Arrays.fill(uptimeInMillis, UNKNOWN_UPTIME); } - public void withShardWriteLoad(int shardId, double load, long uptimeInMillis) { + public Builder withShardWriteLoad(int shardId, double load, long uptimeInMillis) { if (shardId >= this.shardWriteLoad.length) { throw new IllegalArgumentException(); } this.shardWriteLoad[shardId] = load; this.uptimeInMillis[shardId] = uptimeInMillis; + + return this; } public IndexWriteLoad build() { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoadForecast.java b/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoadForecast.java new file mode 100644 index 0000000000000..a7956ac574791 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoadForecast.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.OptionalDouble; +import java.util.OptionalLong; + +public class IndexWriteLoadForecast implements Writeable, ToXContentFragment { + public static final ParseField SHARDS_WRITE_LOAD_FIELD = new ParseField("loads"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "index_write_load_forecast_parser", + false, + (args, unused) -> new IndexWriteLoadForecast(((List) args[0]).stream().mapToDouble(value -> value).toArray()) + ); + + static { + PARSER.declareDoubleArray(ConstructingObjectParser.constructorArg(), SHARDS_WRITE_LOAD_FIELD); + } + + private final double[] forecastedShardWriteLoad; + + private IndexWriteLoadForecast(double[] forecastedShardWriteLoad) { + this.forecastedShardWriteLoad = forecastedShardWriteLoad; + } + + public IndexWriteLoadForecast(StreamInput in) throws IOException { + this(in.readDoubleArray()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDoubleArray(forecastedShardWriteLoad); + } + + public static ClusterState maybeIncludeWriteLoadForecast( + IndexAbstraction.DataStream dataStream, + ClusterState clusterState, + TimeValue maxIndexAge, + TimeValue minShardUptime + ) { + final IndexMetadata writeIndex = clusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + if (IndexSettings.FORECAST_WRITE_LOAD_SETTING.get(writeIndex.getSettings()) == false) { + return clusterState; + } + + final int numberOfShards = writeIndex.getNumberOfShards(); + final double[] totalWriteLoad = new double[numberOfShards]; + final double[] maxWriteLoad = new double[numberOfShards]; + final int[] numberOfIndicesTookIntoAccount = new int[numberOfShards]; + + for (Index index : dataStream.getIndices()) { + final IndexMetadata indexMetadata = clusterState.metadata().getIndexSafe(index); + final long indexAge = System.currentTimeMillis() - indexMetadata.getCreationDate(); + if (index.equals(dataStream.getWriteIndex()) || indexAge > maxIndexAge.millis()) { + continue; + } + + final IndexWriteLoad writeLoad = indexMetadata.getWriteLoad(); + for (int shardId = 0; shardId < numberOfShards; shardId++) { + OptionalDouble writeLoadForShard = writeLoad.getWriteLoadForShard(shardId); + OptionalLong uptimeInMillisForShard = writeLoad.getUptimeInMillisForShard(shardId); + if (writeLoadForShard.isPresent()) { + assert uptimeInMillisForShard.isPresent(); + double shardWriteLoad = writeLoadForShard.getAsDouble(); + long shardUptimeInMillis = uptimeInMillisForShard.getAsLong(); + if (shardUptimeInMillis > minShardUptime.millis()) { + totalWriteLoad[shardId] += shardWriteLoad; + maxWriteLoad[shardId] = Math.max(shardWriteLoad, maxWriteLoad[shardId]); + numberOfIndicesTookIntoAccount[shardId] += 1; + } + } + } + } + + IndexWriteLoadForecast.Builder projectedWriteLoad = IndexWriteLoadForecast.builder(numberOfShards); + boolean modified = false; + for (int shardId = 0; shardId < totalWriteLoad.length; shardId++) { + int shardIndicesTookIntoAccount = numberOfIndicesTookIntoAccount[shardId]; + if (shardIndicesTookIntoAccount > 0) { + modified = true; + double normalizedShardLoad = (totalWriteLoad[shardId] / shardIndicesTookIntoAccount) / maxWriteLoad[shardId]; + projectedWriteLoad.withForecastedWriteLoad(shardId, normalizedShardLoad); + } + } + + if (modified == false) { + return clusterState; + } + + IndexWriteLoadForecast indexWriteLoadForecast = projectedWriteLoad.build(); + + final IndexMetadata updatedWriteIndexMetadata = IndexMetadata.builder(writeIndex) + .indexWriteLoadForecast(indexWriteLoadForecast) + .build(); + + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).put(updatedWriteIndexMetadata, false)) + .build(); + } + + public static IndexWriteLoadForecast fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.field(SHARDS_WRITE_LOAD_FIELD.getPreferredName(), forecastedShardWriteLoad); + } + + public OptionalDouble getForecastedWriteLoadForShard(int shardId) { + double writeLoadForecast = forecastedShardWriteLoad[shardId]; + return writeLoadForecast == -1.0 ? OptionalDouble.empty() : OptionalDouble.of(writeLoadForecast); + } + + public static Builder builder(int numberOfShards) { + return new Builder(numberOfShards); + } + + public static class Builder { + private final double[] shardWriteLoadForecast; + + public Builder(int numberOfShards) { + this.shardWriteLoadForecast = new double[numberOfShards]; + Arrays.fill(shardWriteLoadForecast, -1); + } + + public void withForecastedWriteLoad(int shardId, double shardLoad) { + shardWriteLoadForecast[shardId] = shardLoad; + } + + public IndexWriteLoadForecast build() { + return new IndexWriteLoadForecast(shardWriteLoadForecast); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadForecastTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadForecastTests.java new file mode 100644 index 0000000000000..6dbe3dc7d277e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadForecastTests.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.OptionalDouble; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class IndexWriteLoadForecastTests extends ESTestCase { + public void testForecastComputation() { + final var metadataBuilder = Metadata.builder(); + final int numberOfBackingIndices = 10; + final int numberOfShards = 1; + final List backingIndices = new ArrayList<>(); + final String dataStreamName = "my-ds"; + for (int i = 0; i < numberOfBackingIndices; i++) { + String dataStreamBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, i); + IndexMetadata indexMetadata = IndexMetadata.builder(dataStreamBackingIndexName) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexSettings.FORECAST_WRITE_LOAD_SETTING.getKey(), true) + .build() + ) + .indexWriteLoad( + IndexWriteLoad.builder(numberOfShards).withShardWriteLoad(0, 12, TimeValue.timeValueHours(24).millis()).build() + ) + .creationDate(System.currentTimeMillis()) + .build(); + backingIndices.add(indexMetadata.getIndex()); + metadataBuilder.put(indexMetadata, false); + } + var ds = new DataStream( + dataStreamName, + backingIndices, + numberOfBackingIndices, + Collections.emptyMap(), + false, + false, + false, + false, + IndexMode.STANDARD + ); + metadataBuilder.put(ds); + final var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); + final IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) clusterState.metadata() + .getIndicesLookup() + .get(dataStreamName); + + final var updatedClusterState = IndexWriteLoadForecast.maybeIncludeWriteLoadForecast( + dataStream, + clusterState, + TimeValue.timeValueDays(7), + TimeValue.timeValueHours(8) + ); + + IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + OptionalDouble forecastedWriteLoadForShard = writeIndex.getForecastedWriteLoadForShard(0); + + assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); + assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(1.0))); + } + + public void testForecastComputationFallbacksToSetting() { + final var metadataBuilder = Metadata.builder(); + final int numberOfBackingIndices = 10; + final int numberOfShards = 1; + final List backingIndices = new ArrayList<>(); + final String dataStreamName = "my-ds"; + for (int i = 0; i < numberOfBackingIndices; i++) { + String dataStreamBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, i); + IndexMetadata indexMetadata = IndexMetadata.builder(dataStreamBackingIndexName) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexSettings.FORECAST_WRITE_LOAD_SETTING.getKey(), true) + .put(IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING.getKey(), 0.5) + .build() + ) + .indexWriteLoad( + IndexWriteLoad.builder(numberOfShards).withShardWriteLoad(0, 12, TimeValue.timeValueHours(24).millis()).build() + ) + .creationDate(System.currentTimeMillis()) + .build(); + backingIndices.add(indexMetadata.getIndex()); + metadataBuilder.put(indexMetadata, false); + } + var ds = new DataStream( + dataStreamName, + backingIndices, + numberOfBackingIndices, + Collections.emptyMap(), + false, + false, + false, + false, + IndexMode.STANDARD + ); + metadataBuilder.put(ds); + final var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); + final IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) clusterState.metadata() + .getIndicesLookup() + .get(dataStreamName); + + final var updatedClusterState = IndexWriteLoadForecast.maybeIncludeWriteLoadForecast( + dataStream, + clusterState, + TimeValue.timeValueDays(7), + TimeValue.timeValueHours(8) + ); + + IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + OptionalDouble forecastedWriteLoadForShard = writeIndex.getForecastedWriteLoadForShard(0); + + assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); + assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(0.5))); + } + +} diff --git a/x-pack/plugin/write-load-plugin/build.gradle b/x-pack/plugin/write-load-plugin/build.gradle new file mode 100644 index 0000000000000..662774c598673 --- /dev/null +++ b/x-pack/plugin/write-load-plugin/build.gradle @@ -0,0 +1,21 @@ +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + name 'x-pack-write-load' + description 'x' + classname 'org.elasticsearch.xpack.writeload.WriteLoadPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-write-load' + +dependencies { + compileOnly project(path: xpackModule('core')) + testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(':modules:data-streams') +} + +testClusters.configureEach { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' +} diff --git a/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadForecastIndexSettingProvider.java b/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadForecastIndexSettingProvider.java new file mode 100644 index 0000000000000..d84e41d2b1e23 --- /dev/null +++ b/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadForecastIndexSettingProvider.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.writeload; + +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettings; + +import java.time.Instant; +import java.util.List; +import java.util.function.BooleanSupplier; + +class WriteLoadForecastIndexSettingProvider implements IndexSettingProvider { + private final BooleanSupplier hasValidLicense; + + WriteLoadForecastIndexSettingProvider(BooleanSupplier hasValidLicense) { + this.hasValidLicense = hasValidLicense; + } + + @Override + public Settings getAdditionalIndexSettings( + String indexName, + String dataStreamName, + boolean timeSeries, + Metadata metadata, + Instant resolvedAt, + Settings allSettings, + List combinedTemplateMappings + ) { + if (dataStreamName != null && metadata.dataStreams().get(dataStreamName) != null && hasValidLicense.getAsBoolean()) { + var settingsBuilder = Settings.builder().put(IndexSettings.FORECAST_WRITE_LOAD_SETTING.getKey(), true); + if (allSettings.hasValue(IndexSettings.DEFAULT_WRITE_LOAD_SETTING.getKey())) { + // TODO: warn when the setting exists and the license is invalid? + settingsBuilder.put( + IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING.getKey(), + IndexSettings.DEFAULT_WRITE_LOAD_SETTING.get(allSettings) + ); + } + return settingsBuilder.build(); + } else { + return Settings.EMPTY; + } + } +} diff --git a/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadPlugin.java b/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadPlugin.java new file mode 100644 index 0000000000000..1a43dd22f7091 --- /dev/null +++ b/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadPlugin.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.writeload; + +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicensedFeature; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.XPackPlugin; + +import java.util.Collection; +import java.util.List; + +public class WriteLoadPlugin extends Plugin { + public static final LicensedFeature.Momentary WRITE_LOAD_FORECAST_FEATURE = LicensedFeature.momentary( + null, + "write-load-forecast", + License.OperationMode.ENTERPRISE + ); + + public WriteLoadPlugin() {} + + protected boolean hasValidLicense() { + return WRITE_LOAD_FORECAST_FEATURE.check(XPackPlugin.getSharedLicenseState()); + } + + @Override + public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { + return List.of(new WriteLoadForecastIndexSettingProvider(this::hasValidLicense)); + } +} From 52899e0e27fc2952fa9ea59b80880ab845ff1b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 8 Nov 2022 18:52:14 +0100 Subject: [PATCH 02/10] Update docs/changelog/91425.yaml --- docs/changelog/91425.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/91425.yaml diff --git a/docs/changelog/91425.yaml b/docs/changelog/91425.yaml new file mode 100644 index 0000000000000..5095b2479e745 --- /dev/null +++ b/docs/changelog/91425.yaml @@ -0,0 +1,5 @@ +pr: 91425 +summary: "[WIP] Forecast normalized write load during rollovers" +area: Allocation +type: enhancement +issues: [] From 2bf6e24170d7b78f22352e2e861972c12f112410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Wed, 9 Nov 2022 12:06:32 +0100 Subject: [PATCH 03/10] Move logic to plugin --- .../DataStreamGetWriteIndexTests.java | 4 +- .../rollover/MetadataRolloverService.java | 15 +- .../elasticsearch/cluster/ClusterModule.java | 14 +- .../cluster/metadata/IndexMetadata.java | 35 ++-- .../allocation/WriteLoadForecaster.java | 34 ++++ .../allocator/BalancedShardsAllocator.java | 7 +- .../common/settings/IndexScopedSettings.java | 4 - .../elasticsearch/index/IndexSettings.java | 27 --- .../index/shard/IndexWriteLoadForecast.java | 162 ------------------ .../java/org/elasticsearch/node/Node.java | 23 ++- .../elasticsearch/plugins/ClusterPlugin.java | 5 + .../MetadataRolloverServiceTests.java | 4 +- .../TransportRolloverActionTests.java | 4 +- .../cluster/ClusterModuleTests.java | 37 +++- .../allocation/BalanceConfigurationTests.java | 2 +- .../metadata/DataStreamTestHelper.java | 9 +- .../build.gradle | 6 +- .../LicensedWriteLoadForecaster.java | 143 ++++++++++++++++ .../WriteLoadForecasterPlugin.java | 57 ++++++ .../LicensedWriteLoadForecasterTests.java | 60 +++---- ...WriteLoadForecastIndexSettingProvider.java | 51 ------ .../xpack/writeload/WriteLoadPlugin.java | 36 ---- 22 files changed, 373 insertions(+), 366 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java delete mode 100644 server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoadForecast.java rename x-pack/plugin/{write-load-plugin => write-load-forecaster}/build.gradle (73%) create mode 100644 x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java create mode 100644 x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java rename server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadForecastTests.java => x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java (72%) delete mode 100644 x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadForecastIndexSettingProvider.java delete mode 100644 x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadPlugin.java diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index 47b0da425db88..b7aceb6b0b46e 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; @@ -274,7 +275,8 @@ public void setup() throws Exception { testThreadPool, createIndexService, indexAliasesService, - EmptySystemIndices.INSTANCE + EmptySystemIndices.INSTANCE, + WriteLoadForecaster.DEFAULT ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index d1aad0af2fa5d..2bd4314bd9e41 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -24,15 +24,14 @@ import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.IndexWriteLoad; -import org.elasticsearch.index.shard.IndexWriteLoadForecast; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.snapshots.SnapshotInProgressException; @@ -66,18 +65,21 @@ public class MetadataRolloverService { private final MetadataCreateIndexService createIndexService; private final MetadataIndexAliasesService indexAliasesService; private final SystemIndices systemIndices; + private final WriteLoadForecaster writeLoadForecaster; @Inject public MetadataRolloverService( ThreadPool threadPool, MetadataCreateIndexService createIndexService, MetadataIndexAliasesService indexAliasesService, - SystemIndices systemIndices + SystemIndices systemIndices, + WriteLoadForecaster writeLoadForecaster ) { this.threadPool = threadPool; this.createIndexService = createIndexService; this.indexAliasesService = indexAliasesService; this.systemIndices = systemIndices; + this.writeLoadForecaster = writeLoadForecaster; } public record RolloverResult(String rolloverIndexName, String sourceIndexName, ClusterState clusterState) { @@ -309,12 +311,7 @@ private RolloverResult rolloverDataStream( ) .build(); - newState = IndexWriteLoadForecast.maybeIncludeWriteLoadForecast( - dataStream, - newState, - TimeValue.timeValueDays(7), - TimeValue.timeValueHours(8) - ); + newState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStreamName, newState); return new RolloverResult(newWriteIndexName, originalWriteIndex.getName(), newState); } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index d7d432c46239f..cf668ee18179e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator; import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator.DesiredBalanceReconcilerAction; @@ -122,7 +123,8 @@ public ClusterModule( SnapshotsInfoService snapshotsInfoService, ThreadPool threadPool, SystemIndices systemIndices, - Supplier rerouteServiceSupplier + Supplier rerouteServiceSupplier, + WriteLoadForecaster writeLoadForecaster ) { this.clusterPlugins = clusterPlugins; this.deciderList = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins); @@ -133,7 +135,8 @@ public ClusterModule( threadPool, clusterPlugins, clusterService, - this::reconcile + this::reconcile, + writeLoadForecaster ); this.clusterService = clusterService; this.indexNameExpressionResolver = new IndexNameExpressionResolver(threadPool.getThreadContext(), systemIndices); @@ -344,14 +347,15 @@ private static ShardsAllocator createShardsAllocator( ThreadPool threadPool, List clusterPlugins, ClusterService clusterService, - DesiredBalanceReconcilerAction reconciler + DesiredBalanceReconcilerAction reconciler, + WriteLoadForecaster writeLoadForecaster ) { Map> allocators = new HashMap<>(); - allocators.put(BALANCED_ALLOCATOR, () -> new BalancedShardsAllocator(settings, clusterSettings)); + allocators.put(BALANCED_ALLOCATOR, () -> new BalancedShardsAllocator(settings, clusterSettings, writeLoadForecaster)); allocators.put( DESIRED_BALANCE_ALLOCATOR, () -> new DesiredBalanceShardsAllocator( - new BalancedShardsAllocator(settings, clusterSettings), + new BalancedShardsAllocator(settings, clusterSettings, writeLoadForecaster), threadPool, clusterService, reconciler diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 724954ac1050b..3150ee8315c73 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -44,7 +44,6 @@ import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.IndexWriteLoad; -import org.elasticsearch.index.shard.IndexWriteLoadForecast; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.rest.RestStatus; @@ -613,7 +612,7 @@ public Iterator> settings() { @Nullable private final IndexWriteLoad writeLoad; @Nullable - private final IndexWriteLoadForecast writeLoadForecast; + private final Double writeLoadForecast; private IndexMetadata( final Index index, @@ -658,7 +657,7 @@ private IndexMetadata( @Nullable final Instant timeSeriesEnd, final Version indexCompatibilityVersion, @Nullable final IndexWriteLoad writeLoad, - @Nullable final IndexWriteLoadForecast writeLoadForecast + @Nullable final Double writeLoadForecast ) { this.index = index; this.version = version; @@ -1175,12 +1174,8 @@ public IndexWriteLoad getWriteLoad() { return writeLoad; } - public OptionalDouble getForecastedWriteLoadForShard(int shardId) { - if (IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING.exists(settings)) { - return OptionalDouble.of(IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING.get(settings)); - } - - return writeLoadForecast == null ? OptionalDouble.empty() : writeLoadForecast.getForecastedWriteLoadForShard(shardId); + public OptionalDouble getForecastedWriteLoadForShard() { + return writeLoadForecast == null ? OptionalDouble.empty() : OptionalDouble.of(writeLoadForecast); } public static final String INDEX_RESIZE_SOURCE_UUID_KEY = "index.resize.source.uuid"; @@ -1418,7 +1413,7 @@ private static class IndexMetadataDiff implements Diff { private final boolean isSystem; private final IndexLongFieldRange timestampRange; private final IndexWriteLoad indexWriteLoad; - private final IndexWriteLoadForecast indexWriteLoadForecast; + private final Double indexWriteLoadForecast; IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); @@ -1506,7 +1501,7 @@ private static class IndexMetadataDiff implements Diff { timestampRange = IndexLongFieldRange.readFrom(in); if (in.getVersion().onOrAfter(WRITE_LOAD_ADDED)) { indexWriteLoad = in.readOptionalWriteable(IndexWriteLoad::new); - indexWriteLoadForecast = in.readOptionalWriteable(IndexWriteLoadForecast::new); + indexWriteLoadForecast = in.readOptionalDouble(); } else { indexWriteLoad = null; indexWriteLoadForecast = null; @@ -1543,7 +1538,7 @@ public void writeTo(StreamOutput out) throws IOException { timestampRange.writeTo(out); if (out.getVersion().onOrAfter(WRITE_LOAD_ADDED)) { out.writeOptionalWriteable(indexWriteLoad); - out.writeOptionalWriteable(indexWriteLoadForecast); + out.writeOptionalDouble(indexWriteLoadForecast); } } @@ -1637,7 +1632,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function builder.indexWriteLoadForecast(parser.doubleValue()); default -> throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]"); } } else { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java new file mode 100644 index 0000000000000..93325798ab543 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; + +import java.util.OptionalDouble; + +public interface WriteLoadForecaster { + WriteLoadForecaster DEFAULT = new DefaultWriteLoadForecaster(); + + ClusterState withWriteLoadForecastForWriteIndex(String dataStreamName, ClusterState clusterState); + + OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata); + + class DefaultWriteLoadForecaster implements WriteLoadForecaster { + @Override + public ClusterState withWriteLoadForecastForWriteIndex(String dataStreamName, ClusterState clusterState) { + return clusterState; + } + + @Override + public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { + return OptionalDouble.empty(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index f8caf08d11f3c..4b5030d4d4e3d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.Decision; import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type; @@ -96,17 +97,19 @@ public class BalancedShardsAllocator implements ShardsAllocator { private volatile WeightFunction weightFunction; private volatile float threshold; + private final WriteLoadForecaster writeLoadForecaster; public BalancedShardsAllocator(Settings settings) { - this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); + this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), WriteLoadForecaster.DEFAULT); } @Inject - public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSettings) { + public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSettings, WriteLoadForecaster writeLoadForecaster) { setWeightFunction(INDEX_BALANCE_FACTOR_SETTING.get(settings), SHARD_BALANCE_FACTOR_SETTING.get(settings)); setThreshold(THRESHOLD_SETTING.get(settings)); clusterSettings.addSettingsUpdateConsumer(INDEX_BALANCE_FACTOR_SETTING, SHARD_BALANCE_FACTOR_SETTING, this::setWeightFunction); clusterSettings.addSettingsUpdateConsumer(THRESHOLD_SETTING, this::setThreshold); + this.writeLoadForecaster = writeLoadForecaster; } private void setWeightFunction(float indexBalance, float shardBalanceFactor) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index e0e0df7c1ebca..0743569359142 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -183,10 +183,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.TIME_SERIES_START_TIME, IndexSettings.TIME_SERIES_END_TIME, - IndexSettings.FORECAST_WRITE_LOAD_SETTING, - IndexSettings.DEFAULT_WRITE_LOAD_SETTING, - IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING, - // Legacy index settings we must keep around for BWC from 7.x EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, IndexMetadata.INDEX_ROLLUP_SOURCE_NAME, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index ef7ce30480f16..dfbfb47937b20 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -564,33 +564,6 @@ public Iterator> settings() { Property.IndexSettingDeprecatedInV7AndRemovedInV8 ); - public static final Setting FORECAST_WRITE_LOAD_SETTING = Setting.boolSetting( - "index.forecast_write_load", - false, - Property.Dynamic, - Property.IndexScope, - Property.PrivateIndex - ); - - public static final Setting DEFAULT_WRITE_LOAD_SETTING = Setting.doubleSetting( - "index.default_write_load", - 0.0, - 0.0, - 1.0, - Property.Dynamic, - Property.IndexScope - ); - - public static final Setting DEFAULT_INTERNAL_WRITE_LOAD_SETTING = Setting.doubleSetting( - "index.internal_default_write_load", - 0.0, - 0.0, - 1.0, - Property.Dynamic, - Property.IndexScope, - Property.PrivateIndex - ); - private final Index index; private final Version version; private final Logger logger; diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoadForecast.java b/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoadForecast.java deleted file mode 100644 index a7956ac574791..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoadForecast.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.shard; - -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentFragment; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.OptionalDouble; -import java.util.OptionalLong; - -public class IndexWriteLoadForecast implements Writeable, ToXContentFragment { - public static final ParseField SHARDS_WRITE_LOAD_FIELD = new ParseField("loads"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "index_write_load_forecast_parser", - false, - (args, unused) -> new IndexWriteLoadForecast(((List) args[0]).stream().mapToDouble(value -> value).toArray()) - ); - - static { - PARSER.declareDoubleArray(ConstructingObjectParser.constructorArg(), SHARDS_WRITE_LOAD_FIELD); - } - - private final double[] forecastedShardWriteLoad; - - private IndexWriteLoadForecast(double[] forecastedShardWriteLoad) { - this.forecastedShardWriteLoad = forecastedShardWriteLoad; - } - - public IndexWriteLoadForecast(StreamInput in) throws IOException { - this(in.readDoubleArray()); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDoubleArray(forecastedShardWriteLoad); - } - - public static ClusterState maybeIncludeWriteLoadForecast( - IndexAbstraction.DataStream dataStream, - ClusterState clusterState, - TimeValue maxIndexAge, - TimeValue minShardUptime - ) { - final IndexMetadata writeIndex = clusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); - if (IndexSettings.FORECAST_WRITE_LOAD_SETTING.get(writeIndex.getSettings()) == false) { - return clusterState; - } - - final int numberOfShards = writeIndex.getNumberOfShards(); - final double[] totalWriteLoad = new double[numberOfShards]; - final double[] maxWriteLoad = new double[numberOfShards]; - final int[] numberOfIndicesTookIntoAccount = new int[numberOfShards]; - - for (Index index : dataStream.getIndices()) { - final IndexMetadata indexMetadata = clusterState.metadata().getIndexSafe(index); - final long indexAge = System.currentTimeMillis() - indexMetadata.getCreationDate(); - if (index.equals(dataStream.getWriteIndex()) || indexAge > maxIndexAge.millis()) { - continue; - } - - final IndexWriteLoad writeLoad = indexMetadata.getWriteLoad(); - for (int shardId = 0; shardId < numberOfShards; shardId++) { - OptionalDouble writeLoadForShard = writeLoad.getWriteLoadForShard(shardId); - OptionalLong uptimeInMillisForShard = writeLoad.getUptimeInMillisForShard(shardId); - if (writeLoadForShard.isPresent()) { - assert uptimeInMillisForShard.isPresent(); - double shardWriteLoad = writeLoadForShard.getAsDouble(); - long shardUptimeInMillis = uptimeInMillisForShard.getAsLong(); - if (shardUptimeInMillis > minShardUptime.millis()) { - totalWriteLoad[shardId] += shardWriteLoad; - maxWriteLoad[shardId] = Math.max(shardWriteLoad, maxWriteLoad[shardId]); - numberOfIndicesTookIntoAccount[shardId] += 1; - } - } - } - } - - IndexWriteLoadForecast.Builder projectedWriteLoad = IndexWriteLoadForecast.builder(numberOfShards); - boolean modified = false; - for (int shardId = 0; shardId < totalWriteLoad.length; shardId++) { - int shardIndicesTookIntoAccount = numberOfIndicesTookIntoAccount[shardId]; - if (shardIndicesTookIntoAccount > 0) { - modified = true; - double normalizedShardLoad = (totalWriteLoad[shardId] / shardIndicesTookIntoAccount) / maxWriteLoad[shardId]; - projectedWriteLoad.withForecastedWriteLoad(shardId, normalizedShardLoad); - } - } - - if (modified == false) { - return clusterState; - } - - IndexWriteLoadForecast indexWriteLoadForecast = projectedWriteLoad.build(); - - final IndexMetadata updatedWriteIndexMetadata = IndexMetadata.builder(writeIndex) - .indexWriteLoadForecast(indexWriteLoadForecast) - .build(); - - return ClusterState.builder(clusterState) - .metadata(Metadata.builder(clusterState.metadata()).put(updatedWriteIndexMetadata, false)) - .build(); - } - - public static IndexWriteLoadForecast fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.field(SHARDS_WRITE_LOAD_FIELD.getPreferredName(), forecastedShardWriteLoad); - } - - public OptionalDouble getForecastedWriteLoadForShard(int shardId) { - double writeLoadForecast = forecastedShardWriteLoad[shardId]; - return writeLoadForecast == -1.0 ? OptionalDouble.empty() : OptionalDouble.of(writeLoadForecast); - } - - public static Builder builder(int numberOfShards) { - return new Builder(numberOfShards); - } - - public static class Builder { - private final double[] shardWriteLoadForecast; - - public Builder(int numberOfShards) { - this.shardWriteLoadForecast = new double[numberOfShards]; - Arrays.fill(shardWriteLoadForecast, -1); - } - - public void withForecastedWriteLoad(int shardId, double shardLoad) { - shardWriteLoadForecast[shardId] = shardLoad; - } - - public IndexWriteLoadForecast build() { - return new IndexWriteLoadForecast(shardWriteLoadForecast); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 0cb75947dfc36..08d703d1ed6c8 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -63,6 +63,7 @@ import org.elasticsearch.cluster.routing.RerouteService; import org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor; import org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -566,6 +567,7 @@ protected Node( repositoriesServiceReference::get, rerouteServiceReference::get ); + final WriteLoadForecaster writeLoadForecaster = getWriteLoadForecaster(settings, clusterService.getClusterSettings()); final ClusterModule clusterModule = new ClusterModule( settings, clusterService, @@ -574,7 +576,8 @@ protected Node( snapshotsInfoService, threadPool, systemIndices, - rerouteServiceReference::get + rerouteServiceReference::get, + writeLoadForecaster ); modules.add(clusterModule); IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class)); @@ -1084,6 +1087,7 @@ protected Node( b.bind(HealthInfoCache.class).toInstance(nodeHealthOverview); b.bind(Tracer.class).toInstance(tracer); b.bind(FileSettingsService.class).toInstance(fileSettingsService); + b.bind(WriteLoadForecaster.class).toInstance(writeLoadForecaster); }); if (ReadinessService.enabled(environment)) { @@ -1233,6 +1237,23 @@ private RecoveryPlannerService getRecoveryPlannerService( return recoveryPlannerPlugins.get(0).createRecoveryPlannerService(shardSnapshotsService); } + private WriteLoadForecaster getWriteLoadForecaster(Settings settings, ClusterSettings clusterSettings) { + final List clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class); + final List writeLoadForecasters = clusterPlugins.stream() + .flatMap(clusterPlugin -> clusterPlugin.createWriteLoadHandler(settings, clusterSettings).stream()) + .toList(); + + if (writeLoadForecasters.isEmpty()) { + return WriteLoadForecaster.DEFAULT; + } + + if (writeLoadForecasters.size() > 1) { + throw new IllegalStateException("A single WriteLoadForecaster was expected but got: " + writeLoadForecasters); + } + + return writeLoadForecasters.get(0); + } + protected TransportService newTransportService( Settings settings, Transport transport, diff --git a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java index 53ed925c48ec3..915abaf58ac42 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java @@ -9,6 +9,7 @@ package org.elasticsearch.plugins; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.common.settings.ClusterSettings; @@ -58,6 +59,10 @@ default Map getExistingShardsAllocators() { return Collections.emptyMap(); } + default Collection createWriteLoadHandler(Settings settings, ClusterSettings clusterSettings) { + return Collections.emptyList(); + } + /** * Called when the node is started */ diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index e95eae894fc6b..9500af1d086f6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; @@ -684,7 +685,8 @@ public void testValidation() throws Exception { null, createIndexService, metadataIndexAliasesService, - EmptySystemIndices.INSTANCE + EmptySystemIndices.INSTANCE, + WriteLoadForecaster.DEFAULT ); String newIndexName = useDataStream == false && randomBoolean() ? "logs-index-9" : null; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 346b58c31cfbf..1a5b058127930 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; @@ -353,7 +354,8 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr mockThreadPool, mockCreateIndexService, mdIndexAliasesService, - EmptySystemIndices.INSTANCE + EmptySystemIndices.INSTANCE, + WriteLoadForecaster.DEFAULT ); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( mockTransportService, diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java index 3ab20566ab74c..fe4147d5f8cb5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; @@ -155,7 +156,14 @@ public void testRegisterAllocationDeciderDuplicate() { public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { return Collections.singletonList(new EnableAllocationDecider(settings, clusterSettings)); } - }), clusterInfoService, null, threadPool, EmptySystemIndices.INSTANCE, ClusterModuleTests::getFakeRerouteService) + }), + clusterInfoService, + null, + threadPool, + EmptySystemIndices.INSTANCE, + ClusterModuleTests::getFakeRerouteService, + WriteLoadForecaster.DEFAULT + ) ); assertEquals(e.getMessage(), "Cannot specify allocation decider [" + EnableAllocationDecider.class.getName() + "] twice"); } @@ -166,7 +174,14 @@ public void testRegisterAllocationDecider() { public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { return Collections.singletonList(new FakeAllocationDecider()); } - }), clusterInfoService, null, threadPool, EmptySystemIndices.INSTANCE, ClusterModuleTests::getFakeRerouteService); + }), + clusterInfoService, + null, + threadPool, + EmptySystemIndices.INSTANCE, + ClusterModuleTests::getFakeRerouteService, + WriteLoadForecaster.DEFAULT + ); assertTrue(module.deciderList.stream().anyMatch(d -> d.getClass().equals(FakeAllocationDecider.class))); } @@ -176,7 +191,14 @@ private ClusterModule newClusterModuleWithShardsAllocator(Settings settings, Str public Map> getShardsAllocators(Settings settings, ClusterSettings clusterSettings) { return Collections.singletonMap(name, supplier); } - }), clusterInfoService, null, threadPool, EmptySystemIndices.INSTANCE, ClusterModuleTests::getFakeRerouteService); + }), + clusterInfoService, + null, + threadPool, + EmptySystemIndices.INSTANCE, + ClusterModuleTests::getFakeRerouteService, + WriteLoadForecaster.DEFAULT + ); } public void testRegisterShardsAllocator() { @@ -205,7 +227,8 @@ public void testUnknownShardsAllocator() { null, threadPool, EmptySystemIndices.INSTANCE, - ClusterModuleTests::getFakeRerouteService + ClusterModuleTests::getFakeRerouteService, + WriteLoadForecaster.DEFAULT ) ); assertEquals("Unknown ShardsAllocator [dne]", e.getMessage()); @@ -263,7 +286,8 @@ public void testRejectsReservedExistingShardsAllocatorName() { null, threadPool, EmptySystemIndices.INSTANCE, - ClusterModuleTests::getFakeRerouteService + ClusterModuleTests::getFakeRerouteService, + WriteLoadForecaster.DEFAULT ); expectThrows(IllegalArgumentException.class, () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator())); } @@ -277,7 +301,8 @@ public void testRejectsDuplicateExistingShardsAllocatorName() { null, threadPool, EmptySystemIndices.INSTANCE, - ClusterModuleTests::getFakeRerouteService + ClusterModuleTests::getFakeRerouteService, + WriteLoadForecaster.DEFAULT ); expectThrows(IllegalArgumentException.class, () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator())); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java index eeed2324dec75..3eb5c353585a1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java @@ -289,7 +289,7 @@ public void testPersistedSettings() { settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), 0.3); settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), 2.0); ClusterSettings service = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - BalancedShardsAllocator allocator = new BalancedShardsAllocator(settings.build(), service); + BalancedShardsAllocator allocator = new BalancedShardsAllocator(settings.build(), service, WriteLoadForecaster.DEFAULT); assertThat(allocator.getIndexBalance(), Matchers.equalTo(0.2f)); assertThat(allocator.getShardBalance(), Matchers.equalTo(0.3f)); assertThat(allocator.getThreshold(), Matchers.equalTo(2.0f)); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index ea8160a3a1601..3d58830ddf491 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -489,7 +490,13 @@ public static MetadataRolloverService getMetadataRolloverService( new IndexSettingProviders(providers) ); MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, null, registry); - return new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, EmptySystemIndices.INSTANCE); + return new MetadataRolloverService( + testThreadPool, + createIndexService, + indexAliasesService, + EmptySystemIndices.INSTANCE, + WriteLoadForecaster.DEFAULT + ); } public static MetadataFieldMapper getDataStreamTimestampFieldMapper() { diff --git a/x-pack/plugin/write-load-plugin/build.gradle b/x-pack/plugin/write-load-forecaster/build.gradle similarity index 73% rename from x-pack/plugin/write-load-plugin/build.gradle rename to x-pack/plugin/write-load-forecaster/build.gradle index 662774c598673..f7114a3d21f31 100644 --- a/x-pack/plugin/write-load-plugin/build.gradle +++ b/x-pack/plugin/write-load-forecaster/build.gradle @@ -2,12 +2,12 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - name 'x-pack-write-load' + name 'x-pack-write-load-forecaster' description 'x' - classname 'org.elasticsearch.xpack.writeload.WriteLoadPlugin' + classname 'org.elasticsearch.xpack.writeloadforecaster.WriteLoadForecasterPlugin' extendedPlugins = ['x-pack-core'] } -archivesBaseName = 'x-pack-write-load' +archivesBaseName = 'x-pack-write-load-forecaster' dependencies { compileOnly project(path: xpackModule('core')) diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java new file mode 100644 index 0000000000000..5c1aaea3fcb38 --- /dev/null +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.writeloadforecaster; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.IndexWriteLoad; + +import java.util.OptionalDouble; +import java.util.OptionalLong; +import java.util.function.BooleanSupplier; + +class LicensedWriteLoadForecaster implements WriteLoadForecaster { + public static final Setting MAX_INDEX_AGE_SETTING = Setting.timeSetting( + "write_load_forecaster.max_index_age", + TimeValue.timeValueDays(7), + TimeValue.timeValueHours(1), + Setting.Property.NodeScope + ); + + public static final Setting MIN_UPTIME_SETTING = Setting.timeSetting( + "write_load_forecaster.min_uptime", + TimeValue.timeValueHours(1), + TimeValue.timeValueMinutes(1), + Setting.Property.NodeScope + ); + + private final BooleanSupplier hasValidLicense; + private volatile TimeValue maxIndexAge; + private volatile TimeValue minShardUptime; + + LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, Settings settings, ClusterSettings clusterSettings) { + this(hasValidLicense, MAX_INDEX_AGE_SETTING.get(settings), MIN_UPTIME_SETTING.get(settings)); + clusterSettings.addSettingsUpdateConsumer(MAX_INDEX_AGE_SETTING, this::setMaxIndexAgeSetting); + clusterSettings.addSettingsUpdateConsumer(MIN_UPTIME_SETTING, this::setMinUptimeSetting); + } + + LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, TimeValue maxIndexAge, TimeValue minShardUptime) { + this.hasValidLicense = hasValidLicense; + this.maxIndexAge = maxIndexAge; + this.minShardUptime = minShardUptime; + } + + private void setMaxIndexAgeSetting(TimeValue updatedMaxIndexAge) { + this.maxIndexAge = updatedMaxIndexAge; + } + + private void setMinUptimeSetting(TimeValue updatedMinShardUptime) { + this.minShardUptime = updatedMinShardUptime; + } + + @Override + public ClusterState withWriteLoadForecastForWriteIndex(String dataStreamName, ClusterState clusterState) { + if (hasValidLicense.getAsBoolean() == false) { + return clusterState; + } + + DataStream dataStream = clusterState.metadata().dataStreams().get(dataStreamName); + + if (dataStream == null) { + return clusterState; + } + + double maxWriteLoadAvg = Double.MIN_VALUE; + double totalWriteLoadAvg = 0; + double totalIndices = 0; + + for (Index index : dataStream.getIndices()) { + final IndexMetadata indexMetadata = clusterState.metadata().getIndexSafe(index); + final long indexAge = System.currentTimeMillis() - indexMetadata.getCreationDate(); + final IndexWriteLoad writeLoad = indexMetadata.getWriteLoad(); + + if (index.equals(dataStream.getWriteIndex()) || indexAge > maxIndexAge.millis() || writeLoad == null) { + continue; + } + + double totalIndexWriteLoad = 0; + int totalShards = 0; + for (int shardId = 0; shardId < indexMetadata.getNumberOfShards(); shardId++) { + final OptionalDouble writeLoadForShard = writeLoad.getWriteLoadForShard(shardId); + final OptionalLong uptimeInMillisForShard = writeLoad.getUptimeInMillisForShard(shardId); + if (writeLoadForShard.isPresent()) { + assert uptimeInMillisForShard.isPresent(); + double shardWriteLoad = writeLoadForShard.getAsDouble(); + long shardUptimeInMillis = uptimeInMillisForShard.getAsLong(); + if (shardUptimeInMillis > minShardUptime.millis()) { + totalIndexWriteLoad += shardWriteLoad; + totalShards++; + } + } + + if (totalShards > 0) { + final double indexWriteLoadAvg = totalIndexWriteLoad / totalShards; + totalWriteLoadAvg += indexWriteLoadAvg; + maxWriteLoadAvg = Math.max(maxWriteLoadAvg, indexWriteLoadAvg); + totalIndices++; + } + } + + } + + if (totalIndices == 0) { + return clusterState; + } + + final double normalizedIndexWriteLoad = (totalWriteLoadAvg / totalIndices) / maxWriteLoadAvg; + final IndexMetadata writeIndex = clusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + + return ClusterState.builder(clusterState) + .metadata( + Metadata.builder(clusterState.metadata()) + .put(IndexMetadata.builder(writeIndex).indexWriteLoadForecast(normalizedIndexWriteLoad)) + ) + .build(); + } + + @Override + public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { + if (hasValidLicense.getAsBoolean() == false) { + return OptionalDouble.empty(); + } + + if (WriteLoadForecasterPlugin.DEFAULT_WRITE_LOAD_FORECAST_SETTING.exists(indexMetadata.getSettings())) { + Double defaultWriteLoad = WriteLoadForecasterPlugin.DEFAULT_WRITE_LOAD_FORECAST_SETTING.get(indexMetadata.getSettings()); + return OptionalDouble.of(defaultWriteLoad); + } + + return indexMetadata.getForecastedWriteLoadForShard(); + } +} diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java new file mode 100644 index 0000000000000..a8f83f40929aa --- /dev/null +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.writeloadforecaster; + +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicensedFeature; +import org.elasticsearch.plugins.ClusterPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.XPackPlugin; + +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.MAX_INDEX_AGE_SETTING; +import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.MIN_UPTIME_SETTING; + +public class WriteLoadForecasterPlugin extends Plugin implements ClusterPlugin { + public static final LicensedFeature.Momentary WRITE_LOAD_FORECAST_FEATURE = LicensedFeature.momentary( + null, + "write-load-forecast", + License.OperationMode.ENTERPRISE + ); + + public static final Setting DEFAULT_WRITE_LOAD_FORECAST_SETTING = Setting.doubleSetting( + "index.default_write_load_forecast", + 0.0, + 0.0, + 1.0, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + public WriteLoadForecasterPlugin() {} + + protected boolean hasValidLicense() { + return WRITE_LOAD_FORECAST_FEATURE.check(XPackPlugin.getSharedLicenseState()); + } + + @Override + public List> getSettings() { + return List.of(MAX_INDEX_AGE_SETTING, MIN_UPTIME_SETTING, DEFAULT_WRITE_LOAD_FORECAST_SETTING); + } + + @Override + public Collection createWriteLoadHandler(Settings settings, ClusterSettings clusterSettings) { + return List.of(new LicensedWriteLoadForecaster(this::hasValidLicense, settings, clusterSettings)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadForecastTests.java b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java similarity index 72% rename from server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadForecastTests.java rename to x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java index 6dbe3dc7d277e..03c4c4858c382 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadForecastTests.java +++ b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java @@ -1,25 +1,24 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.shard; +package org.elasticsearch.xpack.writeloadforecaster; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -30,13 +29,13 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -public class IndexWriteLoadForecastTests extends ESTestCase { +public class LicensedWriteLoadForecasterTests extends ESTestCase { public void testForecastComputation() { final var metadataBuilder = Metadata.builder(); final int numberOfBackingIndices = 10; final int numberOfShards = 1; final List backingIndices = new ArrayList<>(); - final String dataStreamName = "my-ds"; + final String dataStreamName = "logs-es"; for (int i = 0; i < numberOfBackingIndices; i++) { String dataStreamBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, i); IndexMetadata indexMetadata = IndexMetadata.builder(dataStreamBackingIndexName) @@ -45,7 +44,6 @@ public void testForecastComputation() { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexSettings.FORECAST_WRITE_LOAD_SETTING.getKey(), true) .build() ) .indexWriteLoad( @@ -56,7 +54,7 @@ public void testForecastComputation() { backingIndices.add(indexMetadata.getIndex()); metadataBuilder.put(indexMetadata, false); } - var ds = new DataStream( + final DataStream dataStream = new DataStream( dataStreamName, backingIndices, numberOfBackingIndices, @@ -67,21 +65,20 @@ public void testForecastComputation() { false, IndexMode.STANDARD ); - metadataBuilder.put(ds); + metadataBuilder.put(dataStream); final var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); - final IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) clusterState.metadata() - .getIndicesLookup() - .get(dataStreamName); - final var updatedClusterState = IndexWriteLoadForecast.maybeIncludeWriteLoadForecast( - dataStream, - clusterState, + WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster( + () -> true, TimeValue.timeValueDays(7), - TimeValue.timeValueHours(8) + TimeValue.timeValueHours(1) ); + ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); - OptionalDouble forecastedWriteLoadForShard = writeIndex.getForecastedWriteLoadForShard(0); + + OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(1.0))); @@ -92,7 +89,7 @@ public void testForecastComputationFallbacksToSetting() { final int numberOfBackingIndices = 10; final int numberOfShards = 1; final List backingIndices = new ArrayList<>(); - final String dataStreamName = "my-ds"; + final String dataStreamName = "logs-es"; for (int i = 0; i < numberOfBackingIndices; i++) { String dataStreamBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, i); IndexMetadata indexMetadata = IndexMetadata.builder(dataStreamBackingIndexName) @@ -101,8 +98,7 @@ public void testForecastComputationFallbacksToSetting() { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexSettings.FORECAST_WRITE_LOAD_SETTING.getKey(), true) - .put(IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING.getKey(), 0.5) + .put(WriteLoadForecasterPlugin.DEFAULT_WRITE_LOAD_FORECAST_SETTING.getKey(), 0.5) .build() ) .indexWriteLoad( @@ -113,7 +109,7 @@ public void testForecastComputationFallbacksToSetting() { backingIndices.add(indexMetadata.getIndex()); metadataBuilder.put(indexMetadata, false); } - var ds = new DataStream( + final DataStream dataStream = new DataStream( dataStreamName, backingIndices, numberOfBackingIndices, @@ -124,24 +120,22 @@ public void testForecastComputationFallbacksToSetting() { false, IndexMode.STANDARD ); - metadataBuilder.put(ds); + metadataBuilder.put(dataStream); final var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); - final IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) clusterState.metadata() - .getIndicesLookup() - .get(dataStreamName); - final var updatedClusterState = IndexWriteLoadForecast.maybeIncludeWriteLoadForecast( - dataStream, - clusterState, + WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster( + () -> true, TimeValue.timeValueDays(7), - TimeValue.timeValueHours(8) + TimeValue.timeValueHours(1) ); + ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); - OptionalDouble forecastedWriteLoadForShard = writeIndex.getForecastedWriteLoadForShard(0); + + OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(0.5))); } - } diff --git a/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadForecastIndexSettingProvider.java b/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadForecastIndexSettingProvider.java deleted file mode 100644 index d84e41d2b1e23..0000000000000 --- a/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadForecastIndexSettingProvider.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.writeload; - -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettingProvider; -import org.elasticsearch.index.IndexSettings; - -import java.time.Instant; -import java.util.List; -import java.util.function.BooleanSupplier; - -class WriteLoadForecastIndexSettingProvider implements IndexSettingProvider { - private final BooleanSupplier hasValidLicense; - - WriteLoadForecastIndexSettingProvider(BooleanSupplier hasValidLicense) { - this.hasValidLicense = hasValidLicense; - } - - @Override - public Settings getAdditionalIndexSettings( - String indexName, - String dataStreamName, - boolean timeSeries, - Metadata metadata, - Instant resolvedAt, - Settings allSettings, - List combinedTemplateMappings - ) { - if (dataStreamName != null && metadata.dataStreams().get(dataStreamName) != null && hasValidLicense.getAsBoolean()) { - var settingsBuilder = Settings.builder().put(IndexSettings.FORECAST_WRITE_LOAD_SETTING.getKey(), true); - if (allSettings.hasValue(IndexSettings.DEFAULT_WRITE_LOAD_SETTING.getKey())) { - // TODO: warn when the setting exists and the license is invalid? - settingsBuilder.put( - IndexSettings.DEFAULT_INTERNAL_WRITE_LOAD_SETTING.getKey(), - IndexSettings.DEFAULT_WRITE_LOAD_SETTING.get(allSettings) - ); - } - return settingsBuilder.build(); - } else { - return Settings.EMPTY; - } - } -} diff --git a/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadPlugin.java b/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadPlugin.java deleted file mode 100644 index 1a43dd22f7091..0000000000000 --- a/x-pack/plugin/write-load-plugin/src/main/java/org/elasticsearch/xpack/writeload/WriteLoadPlugin.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.writeload; - -import org.elasticsearch.index.IndexSettingProvider; -import org.elasticsearch.license.License; -import org.elasticsearch.license.LicensedFeature; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.xpack.core.XPackPlugin; - -import java.util.Collection; -import java.util.List; - -public class WriteLoadPlugin extends Plugin { - public static final LicensedFeature.Momentary WRITE_LOAD_FORECAST_FEATURE = LicensedFeature.momentary( - null, - "write-load-forecast", - License.OperationMode.ENTERPRISE - ); - - public WriteLoadPlugin() {} - - protected boolean hasValidLicense() { - return WRITE_LOAD_FORECAST_FEATURE.check(XPackPlugin.getSharedLicenseState()); - } - - @Override - public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { - return List.of(new WriteLoadForecastIndexSettingProvider(this::hasValidLicense)); - } -} From 081b7ac219f27290a144e57a02a39e445c8d3344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Wed, 9 Nov 2022 12:30:49 +0100 Subject: [PATCH 04/10] Add dynamic setting flag --- .../writeloadforecaster/LicensedWriteLoadForecaster.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java index 5c1aaea3fcb38..d613d62e38195 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java @@ -28,14 +28,16 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { "write_load_forecaster.max_index_age", TimeValue.timeValueDays(7), TimeValue.timeValueHours(1), - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.Dynamic ); public static final Setting MIN_UPTIME_SETTING = Setting.timeSetting( "write_load_forecaster.min_uptime", TimeValue.timeValueHours(1), TimeValue.timeValueMinutes(1), - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.Dynamic ); private final BooleanSupplier hasValidLicense; From 48e43d229474011fd1ee374b28e98e5f28dd6565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Thu, 10 Nov 2022 12:55:48 +0100 Subject: [PATCH 05/10] Compute weighted avg based on uptime --- .../cluster/metadata/IndexMetadata.java | 2 +- .../plugin/write-load-forecaster/build.gradle | 5 - .../WriteLoadForecasterIT.java | 213 ++++++++++ .../LicensedWriteLoadForecaster.java | 120 +++--- .../WriteLoadForecasterPlugin.java | 8 +- .../LicensedWriteLoadForecasterTests.java | 379 ++++++++++++++---- 6 files changed, 583 insertions(+), 144 deletions(-) create mode 100644 x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 3150ee8315c73..d7f175f8ed559 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -1174,7 +1174,7 @@ public IndexWriteLoad getWriteLoad() { return writeLoad; } - public OptionalDouble getForecastedWriteLoadForShard() { + public OptionalDouble getForecastedWriteLoad() { return writeLoadForecast == null ? OptionalDouble.empty() : OptionalDouble.of(writeLoadForecast); } diff --git a/x-pack/plugin/write-load-forecaster/build.gradle b/x-pack/plugin/write-load-forecaster/build.gradle index f7114a3d21f31..7fb79f06ff35a 100644 --- a/x-pack/plugin/write-load-forecaster/build.gradle +++ b/x-pack/plugin/write-load-forecaster/build.gradle @@ -14,8 +14,3 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(':modules:data-streams') } - -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' -} diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java new file mode 100644 index 0000000000000..8cd428ac2060a --- /dev/null +++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.writeloadforecaster; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.shard.IndexingStats; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; + +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.OptionalDouble; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class WriteLoadForecasterIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class, FakeLicenseWriteLoadForecasterPlugin.class); + } + + @Before + public void ensureValidLicense() { + FakeLicenseWriteLoadForecasterPlugin.setHasValidLicense(true); + } + + public void testWriteLoadForecastGetsPopulatedDuringRollovers() throws Exception { + final String dataStreamName = "logs-es"; + setUpDataStreamWriteDocsAndRollover(dataStreamName); + + final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state(); + final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + final IndexMetadata writeIndexMetadata = clusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + + final OptionalDouble indexMetadataForecastedWriteLoad = writeIndexMetadata.getForecastedWriteLoad(); + assertThat(indexMetadataForecastedWriteLoad.isPresent(), is(equalTo(true))); + assertThat(indexMetadataForecastedWriteLoad.getAsDouble(), is(greaterThanOrEqualTo(0.0))); + + final WriteLoadForecaster writeLoadForecaster = internalCluster().getCurrentMasterNodeInstance(WriteLoadForecaster.class); + final OptionalDouble forecastedWriteLoad = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); + + assertThat(forecastedWriteLoad.isPresent(), is(equalTo(true))); + assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(indexMetadataForecastedWriteLoad.getAsDouble()))); + + FakeLicenseWriteLoadForecasterPlugin.setHasValidLicense(false); + + final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); + assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); + } + + public void testWriteLoadForecastDoesNotGetPopulatedWithInvalidLicense() throws Exception { + FakeLicenseWriteLoadForecasterPlugin.setHasValidLicense(false); + + final String dataStreamName = "logs-es"; + setUpDataStreamWriteDocsAndRollover(dataStreamName); + + final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state(); + final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + final IndexMetadata writeIndexMetadata = clusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + + assertThat(writeIndexMetadata.getForecastedWriteLoad().isPresent(), is(equalTo(false))); + } + + public void testWriteLoadForecastIsOverriddenBySetting() throws Exception { + final double writeLoadForecastOverride = randomDoubleBetween(64, 128, true); + final String dataStreamName = "logs-es"; + setUpDataStreamWriteDocsAndRollover( + dataStreamName, + Settings.builder() + .put(WriteLoadForecasterPlugin.OVERRIDE_WRITE_LOAD_FORECAST_SETTING.getKey(), writeLoadForecastOverride) + .build() + ); + + final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state(); + final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + final IndexMetadata writeIndexMetadata = clusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + + final OptionalDouble indexMetadataForecastedWriteLoad = writeIndexMetadata.getForecastedWriteLoad(); + assertThat(indexMetadataForecastedWriteLoad.isPresent(), is(equalTo(true))); + assertThat(indexMetadataForecastedWriteLoad.getAsDouble(), is(greaterThanOrEqualTo(0.0))); + + final WriteLoadForecaster writeLoadForecaster = internalCluster().getCurrentMasterNodeInstance(WriteLoadForecaster.class); + final OptionalDouble forecastedWriteLoad = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); + + assertThat(forecastedWriteLoad.isPresent(), is(equalTo(true))); + assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(writeLoadForecastOverride))); + assertThat(forecastedWriteLoad.getAsDouble(), is(not(equalTo(indexMetadataForecastedWriteLoad.getAsDouble())))); + + FakeLicenseWriteLoadForecasterPlugin.setHasValidLicense(false); + + final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); + assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); + } + + private void setUpDataStreamWriteDocsAndRollover(String dataStreamName) throws Exception { + setUpDataStreamWriteDocsAndRollover(dataStreamName, Settings.EMPTY); + } + + private void setUpDataStreamWriteDocsAndRollover(String dataStreamName, Settings extraIndexTemplateSettings) throws Exception { + final int numberOfShards = randomIntBetween(1, 5); + final int numberOfReplicas = randomIntBetween(0, 1); + final Settings indexSettings = Settings.builder() + .put(extraIndexTemplateSettings) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) + .build(); + + assertAcked( + client().execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("my-template").indexTemplate( + new ComposableIndexTemplate( + List.of("logs-*"), + new Template(indexSettings, null, null), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(), + null + ) + ) + ).actionGet() + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet()); + + final int numberOfRollovers = randomIntBetween(5, 10); + for (int i = 0; i < numberOfRollovers; i++) { + + assertBusy(() -> { + for (int j = 0; j < 10; j++) { + indexDocs(dataStreamName, randomIntBetween(100, 200)); + } + + final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state(); + final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + final String writeIndex = dataStream.getWriteIndex().getName(); + final IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(writeIndex).get(); + for (IndexShardStats indexShardStats : indicesStatsResponse.getIndex(writeIndex).getIndexShards().values()) { + for (ShardStats shard : indexShardStats.getShards()) { + final IndexingStats.Stats shardIndexingStats = shard.getStats().getIndexing().getTotal(); + // Ensure that we have enough clock granularity before rolling over to ensure that we capture _some_ write load + assertThat(shardIndexingStats.getTotalActiveTimeInMillis(), is(greaterThan(0L))); + assertThat(shardIndexingStats.getWriteLoad(), is(greaterThan(0.0))); + } + } + }); + + assertAcked(client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet()); + } + } + + static void indexDocs(String dataStream, int numDocs) { + BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < numDocs; i++) { + String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + bulkRequest.add( + new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } + client().bulk(bulkRequest).actionGet(); + } + + public static class FakeLicenseWriteLoadForecasterPlugin extends WriteLoadForecasterPlugin { + private static final AtomicBoolean hasValidLicense = new AtomicBoolean(true); + + public FakeLicenseWriteLoadForecasterPlugin() {} + + static void setHasValidLicense(boolean validLicense) { + hasValidLicense.set(validLicense); + } + + @Override + protected boolean hasValidLicense() { + return hasValidLicense.get(); + } + } +} diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java index d613d62e38195..c45882e9ee6a5 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java @@ -19,6 +19,8 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.IndexWriteLoad; +import java.util.List; +import java.util.Objects; import java.util.OptionalDouble; import java.util.OptionalLong; import java.util.function.BooleanSupplier; @@ -31,102 +33,98 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { Setting.Property.NodeScope, Setting.Property.Dynamic ); - - public static final Setting MIN_UPTIME_SETTING = Setting.timeSetting( - "write_load_forecaster.min_uptime", - TimeValue.timeValueHours(1), - TimeValue.timeValueMinutes(1), - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - private final BooleanSupplier hasValidLicense; private volatile TimeValue maxIndexAge; - private volatile TimeValue minShardUptime; LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, Settings settings, ClusterSettings clusterSettings) { - this(hasValidLicense, MAX_INDEX_AGE_SETTING.get(settings), MIN_UPTIME_SETTING.get(settings)); + this(hasValidLicense, MAX_INDEX_AGE_SETTING.get(settings)); clusterSettings.addSettingsUpdateConsumer(MAX_INDEX_AGE_SETTING, this::setMaxIndexAgeSetting); - clusterSettings.addSettingsUpdateConsumer(MIN_UPTIME_SETTING, this::setMinUptimeSetting); } - LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, TimeValue maxIndexAge, TimeValue minShardUptime) { + LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, TimeValue maxIndexAge) { this.hasValidLicense = hasValidLicense; this.maxIndexAge = maxIndexAge; - this.minShardUptime = minShardUptime; } private void setMaxIndexAgeSetting(TimeValue updatedMaxIndexAge) { this.maxIndexAge = updatedMaxIndexAge; } - private void setMinUptimeSetting(TimeValue updatedMinShardUptime) { - this.minShardUptime = updatedMinShardUptime; - } - @Override public ClusterState withWriteLoadForecastForWriteIndex(String dataStreamName, ClusterState clusterState) { if (hasValidLicense.getAsBoolean() == false) { return clusterState; } - DataStream dataStream = clusterState.metadata().dataStreams().get(dataStreamName); + final Metadata metadata = clusterState.metadata(); + final DataStream dataStream = metadata.dataStreams().get(dataStreamName); if (dataStream == null) { return clusterState; } - double maxWriteLoadAvg = Double.MIN_VALUE; - double totalWriteLoadAvg = 0; - double totalIndices = 0; + final List indicesWriteLoadWithinMaxAgeRange = getIndicesWithinMaxAgeRange(dataStream, metadata).stream() + .filter(index -> index.equals(dataStream.getWriteIndex()) == false) + .map(metadata::getIndexSafe) + .map(IndexMetadata::getWriteLoad) + .filter(Objects::nonNull) + .toList(); - for (Index index : dataStream.getIndices()) { - final IndexMetadata indexMetadata = clusterState.metadata().getIndexSafe(index); - final long indexAge = System.currentTimeMillis() - indexMetadata.getCreationDate(); - final IndexWriteLoad writeLoad = indexMetadata.getWriteLoad(); + OptionalDouble forecastIndexWriteLoad = forecastIndexWriteLoad(indicesWriteLoadWithinMaxAgeRange); - if (index.equals(dataStream.getWriteIndex()) || indexAge > maxIndexAge.millis() || writeLoad == null) { - continue; - } + if (forecastIndexWriteLoad.isEmpty()) { + return clusterState; + } + + final IndexMetadata writeIndex = metadata.getIndexSafe(dataStream.getWriteIndex()); - double totalIndexWriteLoad = 0; - int totalShards = 0; - for (int shardId = 0; shardId < indexMetadata.getNumberOfShards(); shardId++) { + return ClusterState.builder(clusterState) + .metadata( + Metadata.builder(metadata) + .put(IndexMetadata.builder(writeIndex).indexWriteLoadForecast(forecastIndexWriteLoad.getAsDouble()).build(), false) + ) + .build(); + } + + // Visible for testing + static OptionalDouble forecastIndexWriteLoad(List indicesWriteLoadWithinMaxAgeRange) { + double totalWeightedWriteLoad = 0; + long totalShardUptime = 0; + for (IndexWriteLoad writeLoad : indicesWriteLoadWithinMaxAgeRange) { + for (int shardId = 0; shardId < writeLoad.numberOfShards(); shardId++) { final OptionalDouble writeLoadForShard = writeLoad.getWriteLoadForShard(shardId); final OptionalLong uptimeInMillisForShard = writeLoad.getUptimeInMillisForShard(shardId); if (writeLoadForShard.isPresent()) { assert uptimeInMillisForShard.isPresent(); double shardWriteLoad = writeLoadForShard.getAsDouble(); long shardUptimeInMillis = uptimeInMillisForShard.getAsLong(); - if (shardUptimeInMillis > minShardUptime.millis()) { - totalIndexWriteLoad += shardWriteLoad; - totalShards++; - } - } - - if (totalShards > 0) { - final double indexWriteLoadAvg = totalIndexWriteLoad / totalShards; - totalWriteLoadAvg += indexWriteLoadAvg; - maxWriteLoadAvg = Math.max(maxWriteLoadAvg, indexWriteLoadAvg); - totalIndices++; + totalWeightedWriteLoad += shardWriteLoad * shardUptimeInMillis; + totalShardUptime += shardUptimeInMillis; } } - - } - - if (totalIndices == 0) { - return clusterState; } - final double normalizedIndexWriteLoad = (totalWriteLoadAvg / totalIndices) / maxWriteLoadAvg; - final IndexMetadata writeIndex = clusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + return totalShardUptime == 0 ? OptionalDouble.empty() : OptionalDouble.of(totalWeightedWriteLoad / totalShardUptime); + } - return ClusterState.builder(clusterState) - .metadata( - Metadata.builder(clusterState.metadata()) - .put(IndexMetadata.builder(writeIndex).indexWriteLoadForecast(normalizedIndexWriteLoad)) - ) - .build(); + // Visible for testing + List getIndicesWithinMaxAgeRange(DataStream dataStream, Metadata metadata) { + final List dataStreamIndices = dataStream.getIndices(); + // Consider at least 1 index (including the write index) for cases where rollovers happen less often than maxIndexAge + int firstIndexWithinAgeRange = Math.max(dataStreamIndices.size() - 2, 0); + for (int i = 0; i < dataStreamIndices.size(); i++) { + Index index = dataStreamIndices.get(i); + final IndexMetadata indexMetadata = metadata.getIndexSafe(index); + final long indexAge = System.currentTimeMillis() - indexMetadata.getCreationDate(); + if (indexAge < maxIndexAge.getMillis()) { + // We need to consider the previous index too in order to cover the entire max-index-age range. + firstIndexWithinAgeRange = i == 0 ? 0 : i - 1; + break; + } + } + return firstIndexWithinAgeRange == 0 + ? dataStreamIndices + : dataStreamIndices.subList(firstIndexWithinAgeRange, dataStreamIndices.size()); } @Override @@ -135,11 +133,13 @@ public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return OptionalDouble.empty(); } - if (WriteLoadForecasterPlugin.DEFAULT_WRITE_LOAD_FORECAST_SETTING.exists(indexMetadata.getSettings())) { - Double defaultWriteLoad = WriteLoadForecasterPlugin.DEFAULT_WRITE_LOAD_FORECAST_SETTING.get(indexMetadata.getSettings()); - return OptionalDouble.of(defaultWriteLoad); + if (WriteLoadForecasterPlugin.OVERRIDE_WRITE_LOAD_FORECAST_SETTING.exists(indexMetadata.getSettings())) { + Double overrideWriteLoadForecast = WriteLoadForecasterPlugin.OVERRIDE_WRITE_LOAD_FORECAST_SETTING.get( + indexMetadata.getSettings() + ); + return OptionalDouble.of(overrideWriteLoadForecast); } - return indexMetadata.getForecastedWriteLoadForShard(); + return indexMetadata.getForecastedWriteLoad(); } } diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java index a8f83f40929aa..d64db89e08038 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java @@ -21,7 +21,6 @@ import java.util.List; import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.MAX_INDEX_AGE_SETTING; -import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.MIN_UPTIME_SETTING; public class WriteLoadForecasterPlugin extends Plugin implements ClusterPlugin { public static final LicensedFeature.Momentary WRITE_LOAD_FORECAST_FEATURE = LicensedFeature.momentary( @@ -30,11 +29,10 @@ public class WriteLoadForecasterPlugin extends Plugin implements ClusterPlugin { License.OperationMode.ENTERPRISE ); - public static final Setting DEFAULT_WRITE_LOAD_FORECAST_SETTING = Setting.doubleSetting( - "index.default_write_load_forecast", + public static final Setting OVERRIDE_WRITE_LOAD_FORECAST_SETTING = Setting.doubleSetting( + "index.override_write_load_forecast", 0.0, 0.0, - 1.0, Setting.Property.Dynamic, Setting.Property.IndexScope ); @@ -47,7 +45,7 @@ protected boolean hasValidLicense() { @Override public List> getSettings() { - return List.of(MAX_INDEX_AGE_SETTING, MIN_UPTIME_SETTING, DEFAULT_WRITE_LOAD_FORECAST_SETTING); + return List.of(MAX_INDEX_AGE_SETTING, OVERRIDE_WRITE_LOAD_FORECAST_SETTING); } @Override diff --git a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java index 03c4c4858c382..c567552a0ba8a 100644 --- a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java +++ b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java @@ -25,94 +25,344 @@ import java.util.Collections; import java.util.List; import java.util.OptionalDouble; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.forecastIndexWriteLoad; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; public class LicensedWriteLoadForecasterTests extends ESTestCase { - public void testForecastComputation() { - final var metadataBuilder = Metadata.builder(); + public void testWriteLoadForecastIsAddedToWriteIndex() { + final TimeValue maxIndexAge = TimeValue.timeValueDays(7); + final AtomicBoolean hasValidLicense = new AtomicBoolean(true); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, maxIndexAge); + + final Metadata.Builder metadataBuilder = Metadata.builder(); + final String dataStreamName = "logs-es"; final int numberOfBackingIndices = 10; - final int numberOfShards = 1; + final int numberOfShards = randomIntBetween(1, 5); final List backingIndices = new ArrayList<>(); - final String dataStreamName = "logs-es"; for (int i = 0; i < numberOfBackingIndices; i++) { - String dataStreamBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, i); - IndexMetadata indexMetadata = IndexMetadata.builder(dataStreamBackingIndexName) - .settings( - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .build() - ) - .indexWriteLoad( - IndexWriteLoad.builder(numberOfShards).withShardWriteLoad(0, 12, TimeValue.timeValueHours(24).millis()).build() - ) - .creationDate(System.currentTimeMillis()) - .build(); + final IndexMetadata indexMetadata = createIndexMetadata( + DataStream.getDefaultBackingIndexName(dataStreamName, i), + numberOfShards, + randomIndexWriteLoad(numberOfShards), + System.currentTimeMillis() - (maxIndexAge.millis() / 2) + ); backingIndices.add(indexMetadata.getIndex()); metadataBuilder.put(indexMetadata, false); } - final DataStream dataStream = new DataStream( - dataStreamName, - backingIndices, - numberOfBackingIndices, - Collections.emptyMap(), - false, - false, - false, - false, - IndexMode.STANDARD + + final IndexMetadata writeIndexMetadata = createIndexMetadata( + DataStream.getDefaultBackingIndexName(dataStreamName, numberOfBackingIndices), + numberOfShards, + null, + System.currentTimeMillis() ); + backingIndices.add(writeIndexMetadata.getIndex()); + metadataBuilder.put(writeIndexMetadata, false); + + final DataStream dataStream = createDataStream(dataStreamName, backingIndices); metadataBuilder.put(dataStream); - final var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); + + final ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + + final IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + + final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + + assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); + assertThat(forecastedWriteLoadForShard.getAsDouble(), is(greaterThan(0.0))); + + hasValidLicense.set(false); + + final OptionalDouble forecastedWriteLoadForShardAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + assertThat(forecastedWriteLoadForShardAfterLicenseChange.isPresent(), is(false)); + } + + public void testUptimeIsUsedToWeightWriteLoad() { + final TimeValue maxIndexAge = TimeValue.timeValueDays(7); + final var metadataBuilder = Metadata.builder(); + final String dataStreamName = "logs-es"; + final int numberOfShards = 5; + final List backingIndices = new ArrayList<>(); + // Weighted avg 14.4 + final IndexMetadata indexMetadata = createIndexMetadata( + DataStream.getDefaultBackingIndexName(dataStreamName, 0), + numberOfShards, + IndexWriteLoad.builder(numberOfShards) + .withShardWriteLoad(0, 12, 80) + .withShardWriteLoad(1, 24, 5) + .withShardWriteLoad(2, 24, 5) + .withShardWriteLoad(3, 24, 5) + .withShardWriteLoad(4, 24, 5) + .build(), + System.currentTimeMillis() - (maxIndexAge.millis() / 2) + ); + backingIndices.add(indexMetadata.getIndex()); + metadataBuilder.put(indexMetadata, false); - WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster( - () -> true, - TimeValue.timeValueDays(7), - TimeValue.timeValueHours(1) + final IndexMetadata writeIndexMetadata = createIndexMetadata( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + numberOfShards, + null, + System.currentTimeMillis() ); + backingIndices.add(writeIndexMetadata.getIndex()); + metadataBuilder.put(writeIndexMetadata, false); - ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + final DataStream dataStream = createDataStream(dataStreamName, backingIndices); + metadataBuilder.put(dataStream); + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); - IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, maxIndexAge); - OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + final ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + + final IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + + final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); - assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(1.0))); + assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(14.4))); } - public void testForecastComputationFallbacksToSetting() { - final var metadataBuilder = Metadata.builder(); + public void testForecastedWriteLoadIsOverriddenBySetting() { + final TimeValue maxIndexAge = TimeValue.timeValueDays(7); + final AtomicBoolean hasValidLicense = new AtomicBoolean(true); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, maxIndexAge); + + final Metadata.Builder metadataBuilder = Metadata.builder(); + final String dataStreamName = "logs-es"; final int numberOfBackingIndices = 10; - final int numberOfShards = 1; + final int numberOfShards = randomIntBetween(1, 5); final List backingIndices = new ArrayList<>(); - final String dataStreamName = "logs-es"; for (int i = 0; i < numberOfBackingIndices; i++) { - String dataStreamBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, i); - IndexMetadata indexMetadata = IndexMetadata.builder(dataStreamBackingIndexName) - .settings( - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(WriteLoadForecasterPlugin.DEFAULT_WRITE_LOAD_FORECAST_SETTING.getKey(), 0.5) + final IndexMetadata indexMetadata = createIndexMetadata( + DataStream.getDefaultBackingIndexName(dataStreamName, i), + numberOfShards, + randomIndexWriteLoad(numberOfShards), + System.currentTimeMillis() - (maxIndexAge.millis() / 2) + ); + backingIndices.add(indexMetadata.getIndex()); + metadataBuilder.put(indexMetadata, false); + } + + final IndexMetadata writeIndexMetadata = createIndexMetadata( + DataStream.getDefaultBackingIndexName(dataStreamName, numberOfBackingIndices), + numberOfShards, + null, + System.currentTimeMillis(), + Settings.builder().put(WriteLoadForecasterPlugin.OVERRIDE_WRITE_LOAD_FORECAST_SETTING.getKey(), 0.6).build() + ); + backingIndices.add(writeIndexMetadata.getIndex()); + metadataBuilder.put(writeIndexMetadata, false); + + final DataStream dataStream = createDataStream(dataStreamName, backingIndices); + metadataBuilder.put(dataStream); + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); + + final ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + + final IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + + final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + + assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); + assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(0.6))); + + hasValidLicense.set(false); + + final OptionalDouble forecastedWriteLoadForShardAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + assertThat(forecastedWriteLoadForShardAfterLicenseChange.isPresent(), is(false)); + } + + public void testWriteLoadForecast() { + { + OptionalDouble writeLoadForecast = forecastIndexWriteLoad(List.of()); + assertThat(writeLoadForecast.isEmpty(), is(true)); + } + + { + OptionalDouble writeLoadForecast = forecastIndexWriteLoad(List.of(IndexWriteLoad.builder(5).build())); + assertThat(writeLoadForecast.isEmpty(), is(true)); + } + + { + OptionalDouble writeLoadForecast = forecastIndexWriteLoad( + List.of(IndexWriteLoad.builder(1).withShardWriteLoad(0, 12, 100).build()) + ); + assertThat(writeLoadForecast.isPresent(), is(true)); + assertThat(writeLoadForecast.getAsDouble(), is(equalTo(12.0))); + } + + { + OptionalDouble writeLoadForecast = forecastIndexWriteLoad( + List.of( + IndexWriteLoad.builder(5) + .withShardWriteLoad(0, 12, 80) + .withShardWriteLoad(1, 24, 5) + .withShardWriteLoad(2, 24, 5) + .withShardWriteLoad(3, 24, 5) + .withShardWriteLoad(4, 24, 5) .build() ) - .indexWriteLoad( - IndexWriteLoad.builder(numberOfShards).withShardWriteLoad(0, 12, TimeValue.timeValueHours(24).millis()).build() + ); + assertThat(writeLoadForecast.isPresent(), is(true)); + assertThat(writeLoadForecast.getAsDouble(), is(equalTo(14.4))); + } + + { + OptionalDouble writeLoadForecast = forecastIndexWriteLoad( + List.of( + IndexWriteLoad.builder(5) + .withShardWriteLoad(0, 12, 80) + .withShardWriteLoad(1, 24, 5) + .withShardWriteLoad(2, 24, 5) + .withShardWriteLoad(3, 24, 5) + .withShardWriteLoad(4, 24, 4) + .build(), + // Since this shard uptime is really low, it doesn't add much to the avg + IndexWriteLoad.builder(1).withShardWriteLoad(0, 120, 1).build() + ) + ); + assertThat(writeLoadForecast.isPresent(), is(true)); + assertThat(writeLoadForecast.getAsDouble(), is(equalTo(15.36))); + } + + { + OptionalDouble writeLoadForecast = forecastIndexWriteLoad( + List.of( + IndexWriteLoad.builder(2).withShardWriteLoad(0, 12, 25).withShardWriteLoad(1, 12, 25).build(), + + IndexWriteLoad.builder(1).withShardWriteLoad(0, 12, 50).build() ) - .creationDate(System.currentTimeMillis()) - .build(); + ); + assertThat(writeLoadForecast.isPresent(), is(true)); + assertThat(writeLoadForecast.getAsDouble(), is(equalTo(12.0))); + } + + { + // All indices have the same uptime, therefore it's just a regular avg + OptionalDouble writeLoadForecast = forecastIndexWriteLoad( + List.of( + IndexWriteLoad.builder(3) + .withShardWriteLoad(0, 25, 1) + .withShardWriteLoad(1, 18, 1) + .withShardWriteLoad(2, 23, 1) + .build(), + + IndexWriteLoad.builder(2).withShardWriteLoad(0, 6, 1).withShardWriteLoad(1, 8, 1).build(), + + IndexWriteLoad.builder(1).withShardWriteLoad(0, 15, 1).build() + ) + ); + assertThat(writeLoadForecast.isPresent(), is(true)); + assertThat(writeLoadForecast.getAsDouble(), is(closeTo(15.83, 0.01))); + } + } + + public void testGetIndicesWithinMaxAgeRange() { + final TimeValue maxIndexAge = TimeValue.timeValueDays(7); + final LicensedWriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, maxIndexAge); + + final Metadata.Builder metadataBuilder = Metadata.builder(); + final int numberOfBackingIndicesOlderThanMinAge = randomIntBetween(0, 10); + final int numberOfBackingIndicesWithinMinAnge = randomIntBetween(0, 10); + final int numberOfShards = 1; + final List backingIndices = new ArrayList<>(); + final String dataStreamName = "logs-es"; + final List backingIndicesOlderThanMinAge = new ArrayList<>(); + for (int i = 0; i < numberOfBackingIndicesOlderThanMinAge; i++) { + long creationDate = System.currentTimeMillis() - maxIndexAge.millis() * 2; + final IndexMetadata indexMetadata = createIndexMetadata( + DataStream.getDefaultBackingIndexName(dataStreamName, backingIndices.size(), creationDate), + numberOfShards, + randomIndexWriteLoad(numberOfShards), + creationDate + ); backingIndices.add(indexMetadata.getIndex()); + backingIndicesOlderThanMinAge.add(indexMetadata.getIndex()); metadataBuilder.put(indexMetadata, false); } - final DataStream dataStream = new DataStream( - dataStreamName, + + final List backingIndicesWithinMinAge = new ArrayList<>(); + for (int i = 0; i < numberOfBackingIndicesWithinMinAnge; i++) { + final long createdAt = System.currentTimeMillis() - (maxIndexAge.getMillis() / 2); + final IndexMetadata indexMetadata = createIndexMetadata( + DataStream.getDefaultBackingIndexName(dataStreamName, backingIndices.size(), createdAt), + numberOfShards, + randomIndexWriteLoad(numberOfShards), + createdAt + ); + backingIndices.add(indexMetadata.getIndex()); + backingIndicesWithinMinAge.add(indexMetadata.getIndex()); + metadataBuilder.put(indexMetadata, false); + } + + final String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, backingIndices.size()); + final IndexMetadata writeIndexMetadata = createIndexMetadata(writeIndexName, numberOfShards, null, System.currentTimeMillis()); + backingIndices.add(writeIndexMetadata.getIndex()); + metadataBuilder.put(writeIndexMetadata, false); + + final DataStream dataStream = createDataStream(dataStreamName, backingIndices); + + metadataBuilder.put(dataStream); + Metadata metadata = metadataBuilder.build(); + + final List indicesWithinMaxAgeRange = writeLoadForecaster.getIndicesWithinMaxAgeRange(dataStream, metadata); + + final List expectedIndicesWithinMaxAgeRange = new ArrayList<>(); + if (numberOfBackingIndicesOlderThanMinAge > 0) { + expectedIndicesWithinMaxAgeRange.add(backingIndicesOlderThanMinAge.get(backingIndicesOlderThanMinAge.size() - 1)); + } + expectedIndicesWithinMaxAgeRange.addAll(backingIndicesWithinMinAge); + expectedIndicesWithinMaxAgeRange.add(writeIndexMetadata.getIndex()); + + assertThat(indicesWithinMaxAgeRange, is(equalTo(expectedIndicesWithinMaxAgeRange))); + } + + private IndexWriteLoad randomIndexWriteLoad(int numberOfShards) { + IndexWriteLoad.Builder builder = IndexWriteLoad.builder(numberOfShards); + for (int shardId = 0; shardId < numberOfShards; shardId++) { + builder.withShardWriteLoad(shardId, randomDoubleBetween(0, 64, true), randomLongBetween(1, 10)); + } + return builder.build(); + } + + private IndexMetadata createIndexMetadata(String indexName, int numberOfShards, IndexWriteLoad indexWriteLoad, long createdAt) { + return createIndexMetadata(indexName, numberOfShards, indexWriteLoad, createdAt, Settings.EMPTY); + } + + private IndexMetadata createIndexMetadata( + String indexName, + int numberOfShards, + IndexWriteLoad indexWriteLoad, + long createdAt, + Settings extraSettings + ) { + return IndexMetadata.builder(indexName) + .settings( + Settings.builder() + .put(extraSettings) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .build() + ) + .indexWriteLoad(indexWriteLoad) + .creationDate(createdAt) + .build(); + } + + private DataStream createDataStream(String name, List backingIndices) { + return new DataStream( + name, backingIndices, - numberOfBackingIndices, + backingIndices.size(), Collections.emptyMap(), false, false, @@ -120,22 +370,5 @@ public void testForecastComputationFallbacksToSetting() { false, IndexMode.STANDARD ); - metadataBuilder.put(dataStream); - final var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); - - WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster( - () -> true, - TimeValue.timeValueDays(7), - TimeValue.timeValueHours(1) - ); - - ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); - - IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); - - OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); - - assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); - assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(0.5))); } } From 629438124fe5db4282cc342fafb463f531cf0cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Thu, 10 Nov 2022 14:48:35 +0100 Subject: [PATCH 06/10] Review comments --- .../resources/forbidden/es-all-signatures.txt | 3 ++ .../rollover/MetadataRolloverService.java | 20 ++++++------- .../allocation/WriteLoadForecaster.java | 8 ++--- .../cluster/ClusterStateTests.java | 10 +++++-- .../cluster/metadata/IndexMetadataTests.java | 6 ++++ .../WriteLoadForecasterIT.java | 2 ++ .../LicensedWriteLoadForecaster.java | 30 ++++++++----------- .../LicensedWriteLoadForecasterTests.java | 29 ++++++++++-------- 8 files changed, 60 insertions(+), 48 deletions(-) diff --git a/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt index bb37ef72a1688..ca1e6d85be984 100644 --- a/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt @@ -62,3 +62,6 @@ org.apache.logging.log4j.message.ParameterizedMessage#(java.lang.String, j org.apache.logging.log4j.message.ParameterizedMessage#(java.lang.String, java.lang.Object[]) org.apache.logging.log4j.message.ParameterizedMessage#(java.lang.String, java.lang.Object) org.apache.logging.log4j.message.ParameterizedMessage#(java.lang.String, java.lang.Object, java.lang.Object) + +@defaultMessage Use WriteLoadForecaster#getForecastedWriteLoad instead +org.elasticsearch.cluster.metadata.IndexMetadata#getForecastedWriteLoad() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 2bd4314bd9e41..17d0b3a5607be 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -300,18 +300,16 @@ private RolloverResult rolloverDataStream( RolloverInfo rolloverInfo = new RolloverInfo(dataStreamName, metConditions, threadPool.absoluteTimeInMillis()); - newState = ClusterState.builder(newState) - .metadata( - Metadata.builder(newState.metadata()) - .put( - IndexMetadata.builder(newState.metadata().index(originalWriteIndex)) - .indexWriteLoad(sourceIndexWriteLoad) - .putRolloverInfo(rolloverInfo) - ) - ) - .build(); + Metadata.Builder metadataBuilder = Metadata.builder(newState.metadata()) + .put( + IndexMetadata.builder(newState.metadata().index(originalWriteIndex)) + .indexWriteLoad(sourceIndexWriteLoad) + .putRolloverInfo(rolloverInfo) + ); + + metadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStreamName, metadataBuilder); - newState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStreamName, newState); + newState = ClusterState.builder(newState).metadata(metadataBuilder).build(); return new RolloverResult(newWriteIndexName, originalWriteIndex.getName(), newState); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java index 93325798ab543..f7749f0db4d04 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java @@ -8,22 +8,22 @@ package org.elasticsearch.cluster.routing.allocation; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import java.util.OptionalDouble; public interface WriteLoadForecaster { WriteLoadForecaster DEFAULT = new DefaultWriteLoadForecaster(); - ClusterState withWriteLoadForecastForWriteIndex(String dataStreamName, ClusterState clusterState); + Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata); OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata); class DefaultWriteLoadForecaster implements WriteLoadForecaster { @Override - public ClusterState withWriteLoadForecastForWriteIndex(String dataStreamName, ClusterState clusterState) { - return clusterState; + public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) { + return metadata; } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index f76b223884703..15d3f39ec5b5c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -286,7 +286,8 @@ public void testToXContent() throws IOException { "write_load": { "loads": [-1.0], "uptimes": [-1] - } + }, + "write_load_forecast" : 8.0 } }, "index-graveyard": { @@ -506,7 +507,8 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "uptimes" : [ -1 ] - } + }, + "write_load_forecast" : 8.0 } }, "index-graveyard" : { @@ -733,7 +735,8 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "uptimes" : [ -1 ] - } + }, + "write_load_forecast" : 8.0 } }, "index-graveyard" : { @@ -923,6 +926,7 @@ private ClusterState buildClusterState() throws IOException { .numberOfReplicas(2) .putRolloverInfo(new RolloverInfo("rolloveAlias", new ArrayList<>(), 1L)) .indexWriteLoad(IndexWriteLoad.builder(1).build()) + .indexWriteLoadForecast(8.0) .build(); return ClusterState.builder(ClusterName.DEFAULT) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index 229133e6ba4d7..8af1d75cd5393 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.index.shard.ShardId; @@ -66,6 +67,7 @@ protected NamedXContentRegistry xContentRegistry() { return new NamedXContentRegistry(IndicesModule.getNamedXContents()); } + @SuppressForbidden(reason = "Use IndexMetadata#getForecastedWriteLoad to ensure that the serialized value is correct") public void testIndexMetadataSerialization() throws IOException { Integer numShard = randomFrom(1, 2, 4, 8, 16); int numberOfReplicas = randomIntBetween(0, 10); @@ -74,6 +76,7 @@ public void testIndexMetadataSerialization() throws IOException { customMap.put(randomAlphaOfLength(5), randomAlphaOfLength(10)); customMap.put(randomAlphaOfLength(10), randomAlphaOfLength(15)); IndexWriteLoad indexWriteLoad = randomBoolean() ? randomWriteLoad(numShard) : null; + Double indexWriteLoadForecast = randomBoolean() ? randomDoubleBetween(0.0, 128, true) : null; IndexMetadata metadata = IndexMetadata.builder("foo") .settings( Settings.builder() @@ -101,6 +104,7 @@ public void testIndexMetadataSerialization() throws IOException { ) ) .indexWriteLoad(indexWriteLoad) + .indexWriteLoadForecast(indexWriteLoadForecast) .build(); assertEquals(system, metadata.isSystem()); @@ -130,6 +134,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata.getCustomData(), expectedCustom); assertEquals(metadata.getCustomData(), fromXContentMeta.getCustomData()); assertEquals(metadata.getWriteLoad(), fromXContentMeta.getWriteLoad()); + assertEquals(metadata.getForecastedWriteLoad(), fromXContentMeta.getForecastedWriteLoad()); final BytesStreamOutput out = new BytesStreamOutput(); metadata.writeTo(out); @@ -151,6 +156,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata.getCustomData(), deserialized.getCustomData()); assertEquals(metadata.isSystem(), deserialized.isSystem()); assertEquals(metadata.getWriteLoad(), deserialized.getWriteLoad()); + assertEquals(metadata.getForecastedWriteLoad(), fromXContentMeta.getForecastedWriteLoad()); } } diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java index 8cd428ac2060a..22aed0423812b 100644 --- a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java +++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.datastreams.DataStreamsPlugin; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.shard.IndexingStats; @@ -46,6 +47,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +@SuppressForbidden(reason = "Uses IndexMetadata#getForecastedWriteLoad to validate the computation") public class WriteLoadForecasterIT extends ESIntegTestCase { @Override diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java index c45882e9ee6a5..15ac19b268fc3 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.writeloadforecaster; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; @@ -15,6 +14,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.IndexWriteLoad; @@ -51,21 +51,20 @@ private void setMaxIndexAgeSetting(TimeValue updatedMaxIndexAge) { } @Override - public ClusterState withWriteLoadForecastForWriteIndex(String dataStreamName, ClusterState clusterState) { + public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) { if (hasValidLicense.getAsBoolean() == false) { - return clusterState; + return metadata; } - final Metadata metadata = clusterState.metadata(); - final DataStream dataStream = metadata.dataStreams().get(dataStreamName); + final DataStream dataStream = metadata.dataStream(dataStreamName); if (dataStream == null) { - return clusterState; + return metadata; } final List indicesWriteLoadWithinMaxAgeRange = getIndicesWithinMaxAgeRange(dataStream, metadata).stream() .filter(index -> index.equals(dataStream.getWriteIndex()) == false) - .map(metadata::getIndexSafe) + .map(metadata::getSafe) .map(IndexMetadata::getWriteLoad) .filter(Objects::nonNull) .toList(); @@ -73,17 +72,13 @@ public ClusterState withWriteLoadForecastForWriteIndex(String dataStreamName, Cl OptionalDouble forecastIndexWriteLoad = forecastIndexWriteLoad(indicesWriteLoadWithinMaxAgeRange); if (forecastIndexWriteLoad.isEmpty()) { - return clusterState; + return metadata; } - final IndexMetadata writeIndex = metadata.getIndexSafe(dataStream.getWriteIndex()); + final IndexMetadata writeIndex = metadata.getSafe(dataStream.getWriteIndex()); + metadata.put(IndexMetadata.builder(writeIndex).indexWriteLoadForecast(forecastIndexWriteLoad.getAsDouble()).build(), false); - return ClusterState.builder(clusterState) - .metadata( - Metadata.builder(metadata) - .put(IndexMetadata.builder(writeIndex).indexWriteLoadForecast(forecastIndexWriteLoad.getAsDouble()).build(), false) - ) - .build(); + return metadata; } // Visible for testing @@ -108,13 +103,13 @@ static OptionalDouble forecastIndexWriteLoad(List indicesWriteLo } // Visible for testing - List getIndicesWithinMaxAgeRange(DataStream dataStream, Metadata metadata) { + List getIndicesWithinMaxAgeRange(DataStream dataStream, Metadata.Builder metadata) { final List dataStreamIndices = dataStream.getIndices(); // Consider at least 1 index (including the write index) for cases where rollovers happen less often than maxIndexAge int firstIndexWithinAgeRange = Math.max(dataStreamIndices.size() - 2, 0); for (int i = 0; i < dataStreamIndices.size(); i++) { Index index = dataStreamIndices.get(i); - final IndexMetadata indexMetadata = metadata.getIndexSafe(index); + final IndexMetadata indexMetadata = metadata.getSafe(index); final long indexAge = System.currentTimeMillis() - indexMetadata.getCreationDate(); if (indexAge < maxIndexAge.getMillis()) { // We need to consider the previous index too in order to cover the entire max-index-age range. @@ -128,6 +123,7 @@ List getIndicesWithinMaxAgeRange(DataStream dataStream, Metadata metadata } @Override + @SuppressForbidden(reason = "This is the only place where IndexMetadata#getForecastedWriteLoad is allowed to be used") public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { if (hasValidLicense.getAsBoolean() == false) { return OptionalDouble.empty(); diff --git a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java index c567552a0ba8a..da221d982fa58 100644 --- a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java +++ b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java @@ -8,8 +8,6 @@ package org.elasticsearch.xpack.writeloadforecaster; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; @@ -66,11 +64,13 @@ public void testWriteLoadForecastIsAddedToWriteIndex() { final DataStream dataStream = createDataStream(dataStreamName, backingIndices); metadataBuilder.put(dataStream); - final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); - final ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + final Metadata.Builder updatedMetadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex( + dataStream.getName(), + metadataBuilder + ); - final IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + final IndexMetadata writeIndex = updatedMetadataBuilder.getSafe(dataStream.getWriteIndex()); final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); @@ -116,13 +116,15 @@ public void testUptimeIsUsedToWeightWriteLoad() { final DataStream dataStream = createDataStream(dataStreamName, backingIndices); metadataBuilder.put(dataStream); - final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, maxIndexAge); - final ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + final Metadata.Builder updatedMetadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex( + dataStream.getName(), + metadataBuilder + ); - final IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + final IndexMetadata writeIndex = updatedMetadataBuilder.getSafe(dataStream.getWriteIndex()); final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); @@ -163,11 +165,13 @@ public void testForecastedWriteLoadIsOverriddenBySetting() { final DataStream dataStream = createDataStream(dataStreamName, backingIndices); metadataBuilder.put(dataStream); - final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); - final ClusterState updatedClusterState = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStream.getName(), clusterState); + final Metadata.Builder updatedMetadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex( + dataStream.getName(), + metadataBuilder + ); - final IndexMetadata writeIndex = updatedClusterState.metadata().getIndexSafe(dataStream.getWriteIndex()); + final IndexMetadata writeIndex = updatedMetadataBuilder.getSafe(dataStream.getWriteIndex()); final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); @@ -311,9 +315,8 @@ public void testGetIndicesWithinMaxAgeRange() { final DataStream dataStream = createDataStream(dataStreamName, backingIndices); metadataBuilder.put(dataStream); - Metadata metadata = metadataBuilder.build(); - final List indicesWithinMaxAgeRange = writeLoadForecaster.getIndicesWithinMaxAgeRange(dataStream, metadata); + final List indicesWithinMaxAgeRange = writeLoadForecaster.getIndicesWithinMaxAgeRange(dataStream, metadataBuilder); final List expectedIndicesWithinMaxAgeRange = new ArrayList<>(); if (numberOfBackingIndicesOlderThanMinAge > 0) { From 0655ab42bddbb2d82bfd0798118490da64c183c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Mon, 14 Nov 2022 11:46:04 +0100 Subject: [PATCH 07/10] Review comments --- .../java/org/elasticsearch/node/Node.java | 10 ++-- .../elasticsearch/plugins/ClusterPlugin.java | 7 ++- .../WriteLoadForecasterIT.java | 26 ++++++---- .../LicensedWriteLoadForecaster.java | 22 +++++--- .../WriteLoadForecasterPlugin.java | 9 +++- .../LicensedWriteLoadForecasterTests.java | 51 ++++++++++++------- 6 files changed, 86 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 08d703d1ed6c8..44ad681a3f4f4 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -567,7 +567,11 @@ protected Node( repositoriesServiceReference::get, rerouteServiceReference::get ); - final WriteLoadForecaster writeLoadForecaster = getWriteLoadForecaster(settings, clusterService.getClusterSettings()); + final WriteLoadForecaster writeLoadForecaster = getWriteLoadForecaster( + threadPool, + settings, + clusterService.getClusterSettings() + ); final ClusterModule clusterModule = new ClusterModule( settings, clusterService, @@ -1237,10 +1241,10 @@ private RecoveryPlannerService getRecoveryPlannerService( return recoveryPlannerPlugins.get(0).createRecoveryPlannerService(shardSnapshotsService); } - private WriteLoadForecaster getWriteLoadForecaster(Settings settings, ClusterSettings clusterSettings) { + private WriteLoadForecaster getWriteLoadForecaster(ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings) { final List clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class); final List writeLoadForecasters = clusterPlugins.stream() - .flatMap(clusterPlugin -> clusterPlugin.createWriteLoadHandler(settings, clusterSettings).stream()) + .flatMap(clusterPlugin -> clusterPlugin.createWriteLoadHandler(threadPool, settings, clusterSettings).stream()) .toList(); if (writeLoadForecasters.isEmpty()) { diff --git a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java index 915abaf58ac42..8e0d23912d935 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; import java.util.Collection; import java.util.Collections; @@ -59,7 +60,11 @@ default Map getExistingShardsAllocators() { return Collections.emptyMap(); } - default Collection createWriteLoadHandler(Settings settings, ClusterSettings clusterSettings) { + default Collection createWriteLoadHandler( + ThreadPool threadPool, + Settings settings, + ClusterSettings clusterSettings + ) { return Collections.emptyList(); } diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java index 22aed0423812b..bef663671065a 100644 --- a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java +++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; @@ -43,7 +44,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -57,7 +57,7 @@ protected Collection> nodePlugins() { @Before public void ensureValidLicense() { - FakeLicenseWriteLoadForecasterPlugin.setHasValidLicense(true); + setHasValidLicense(true); } public void testWriteLoadForecastGetsPopulatedDuringRollovers() throws Exception { @@ -70,7 +70,7 @@ public void testWriteLoadForecastGetsPopulatedDuringRollovers() throws Exception final OptionalDouble indexMetadataForecastedWriteLoad = writeIndexMetadata.getForecastedWriteLoad(); assertThat(indexMetadataForecastedWriteLoad.isPresent(), is(equalTo(true))); - assertThat(indexMetadataForecastedWriteLoad.getAsDouble(), is(greaterThanOrEqualTo(0.0))); + assertThat(indexMetadataForecastedWriteLoad.getAsDouble(), is(greaterThan(0.0))); final WriteLoadForecaster writeLoadForecaster = internalCluster().getCurrentMasterNodeInstance(WriteLoadForecaster.class); final OptionalDouble forecastedWriteLoad = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); @@ -78,14 +78,14 @@ public void testWriteLoadForecastGetsPopulatedDuringRollovers() throws Exception assertThat(forecastedWriteLoad.isPresent(), is(equalTo(true))); assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(indexMetadataForecastedWriteLoad.getAsDouble()))); - FakeLicenseWriteLoadForecasterPlugin.setHasValidLicense(false); + setHasValidLicense(false); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); } public void testWriteLoadForecastDoesNotGetPopulatedWithInvalidLicense() throws Exception { - FakeLicenseWriteLoadForecasterPlugin.setHasValidLicense(false); + setHasValidLicense(false); final String dataStreamName = "logs-es"; setUpDataStreamWriteDocsAndRollover(dataStreamName); @@ -113,7 +113,7 @@ public void testWriteLoadForecastIsOverriddenBySetting() throws Exception { final OptionalDouble indexMetadataForecastedWriteLoad = writeIndexMetadata.getForecastedWriteLoad(); assertThat(indexMetadataForecastedWriteLoad.isPresent(), is(equalTo(true))); - assertThat(indexMetadataForecastedWriteLoad.getAsDouble(), is(greaterThanOrEqualTo(0.0))); + assertThat(indexMetadataForecastedWriteLoad.getAsDouble(), is(greaterThan(0.0))); final WriteLoadForecaster writeLoadForecaster = internalCluster().getCurrentMasterNodeInstance(WriteLoadForecaster.class); final OptionalDouble forecastedWriteLoad = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); @@ -122,7 +122,7 @@ public void testWriteLoadForecastIsOverriddenBySetting() throws Exception { assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(writeLoadForecastOverride))); assertThat(forecastedWriteLoad.getAsDouble(), is(not(equalTo(indexMetadataForecastedWriteLoad.getAsDouble())))); - FakeLicenseWriteLoadForecasterPlugin.setHasValidLicense(false); + setHasValidLicense(false); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); @@ -198,12 +198,20 @@ static void indexDocs(String dataStream, int numDocs) { client().bulk(bulkRequest).actionGet(); } + private void setHasValidLicense(boolean hasValidLicense) { + for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) { + for (var writeLoadForecasterPlugin : pluginsService.filterPlugins(FakeLicenseWriteLoadForecasterPlugin.class)) { + writeLoadForecasterPlugin.setHasValidLicense(hasValidLicense); + } + } + } + public static class FakeLicenseWriteLoadForecasterPlugin extends WriteLoadForecasterPlugin { - private static final AtomicBoolean hasValidLicense = new AtomicBoolean(true); + private final AtomicBoolean hasValidLicense = new AtomicBoolean(true); public FakeLicenseWriteLoadForecasterPlugin() {} - static void setHasValidLicense(boolean validLicense) { + void setHasValidLicense(boolean validLicense) { hasValidLicense.set(validLicense); } diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java index 15ac19b268fc3..7746a2329ff5a 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java @@ -18,6 +18,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.IndexWriteLoad; +import org.elasticsearch.threadpool.ThreadPool; import java.util.List; import java.util.Objects; @@ -34,15 +35,23 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { Setting.Property.Dynamic ); private final BooleanSupplier hasValidLicense; + private final ThreadPool threadPool; private volatile TimeValue maxIndexAge; - LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, Settings settings, ClusterSettings clusterSettings) { - this(hasValidLicense, MAX_INDEX_AGE_SETTING.get(settings)); + LicensedWriteLoadForecaster( + BooleanSupplier hasValidLicense, + ThreadPool threadPool, + Settings settings, + ClusterSettings clusterSettings + ) { + this(hasValidLicense, threadPool, MAX_INDEX_AGE_SETTING.get(settings)); clusterSettings.addSettingsUpdateConsumer(MAX_INDEX_AGE_SETTING, this::setMaxIndexAgeSetting); } - LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, TimeValue maxIndexAge) { + // exposed for tests only + LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, ThreadPool threadPool, TimeValue maxIndexAge) { this.hasValidLicense = hasValidLicense; + this.threadPool = threadPool; this.maxIndexAge = maxIndexAge; } @@ -105,21 +114,20 @@ static OptionalDouble forecastIndexWriteLoad(List indicesWriteLo // Visible for testing List getIndicesWithinMaxAgeRange(DataStream dataStream, Metadata.Builder metadata) { final List dataStreamIndices = dataStream.getIndices(); + final long currentTimeMillis = threadPool.absoluteTimeInMillis(); // Consider at least 1 index (including the write index) for cases where rollovers happen less often than maxIndexAge int firstIndexWithinAgeRange = Math.max(dataStreamIndices.size() - 2, 0); for (int i = 0; i < dataStreamIndices.size(); i++) { Index index = dataStreamIndices.get(i); final IndexMetadata indexMetadata = metadata.getSafe(index); - final long indexAge = System.currentTimeMillis() - indexMetadata.getCreationDate(); + final long indexAge = currentTimeMillis - indexMetadata.getCreationDate(); if (indexAge < maxIndexAge.getMillis()) { // We need to consider the previous index too in order to cover the entire max-index-age range. firstIndexWithinAgeRange = i == 0 ? 0 : i - 1; break; } } - return firstIndexWithinAgeRange == 0 - ? dataStreamIndices - : dataStreamIndices.subList(firstIndexWithinAgeRange, dataStreamIndices.size()); + return dataStreamIndices.subList(firstIndexWithinAgeRange, dataStreamIndices.size()); } @Override diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java index d64db89e08038..99eca1895cd54 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java @@ -15,6 +15,7 @@ import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.XPackPlugin; import java.util.Collection; @@ -49,7 +50,11 @@ public List> getSettings() { } @Override - public Collection createWriteLoadHandler(Settings settings, ClusterSettings clusterSettings) { - return List.of(new LicensedWriteLoadForecaster(this::hasValidLicense, settings, clusterSettings)); + public Collection createWriteLoadHandler( + ThreadPool threadPool, + Settings settings, + ClusterSettings clusterSettings + ) { + return List.of(new LicensedWriteLoadForecaster(this::hasValidLicense, threadPool, settings, clusterSettings)); } } diff --git a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java index da221d982fa58..29f2b44e814b2 100644 --- a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java +++ b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java @@ -18,11 +18,16 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.OptionalDouble; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.forecastIndexWriteLoad; @@ -32,10 +37,22 @@ import static org.hamcrest.Matchers.is; public class LicensedWriteLoadForecasterTests extends ESTestCase { + ThreadPool threadPool; + + @Before + public void setUpThreadPool() { + threadPool = new TestThreadPool(getTestName()); + } + + @After + public void tearDownThreadPool() { + ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); + } + public void testWriteLoadForecastIsAddedToWriteIndex() { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); final AtomicBoolean hasValidLicense = new AtomicBoolean(true); - final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, maxIndexAge); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, maxIndexAge); final Metadata.Builder metadataBuilder = Metadata.builder(); final String dataStreamName = "logs-es"; @@ -72,15 +89,15 @@ public void testWriteLoadForecastIsAddedToWriteIndex() { final IndexMetadata writeIndex = updatedMetadataBuilder.getSafe(dataStream.getWriteIndex()); - final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + final OptionalDouble forecastedWriteLoad = writeLoadForecaster.getForecastedWriteLoad(writeIndex); - assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); - assertThat(forecastedWriteLoadForShard.getAsDouble(), is(greaterThan(0.0))); + assertThat(forecastedWriteLoad.isPresent(), is(true)); + assertThat(forecastedWriteLoad.getAsDouble(), is(greaterThan(0.0))); hasValidLicense.set(false); - final OptionalDouble forecastedWriteLoadForShardAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); - assertThat(forecastedWriteLoadForShardAfterLicenseChange.isPresent(), is(false)); + final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(false)); } public void testUptimeIsUsedToWeightWriteLoad() { @@ -117,7 +134,7 @@ public void testUptimeIsUsedToWeightWriteLoad() { final DataStream dataStream = createDataStream(dataStreamName, backingIndices); metadataBuilder.put(dataStream); - final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, maxIndexAge); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, threadPool, maxIndexAge); final Metadata.Builder updatedMetadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex( dataStream.getName(), @@ -126,16 +143,16 @@ public void testUptimeIsUsedToWeightWriteLoad() { final IndexMetadata writeIndex = updatedMetadataBuilder.getSafe(dataStream.getWriteIndex()); - final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + final OptionalDouble forecastedWriteLoad = writeLoadForecaster.getForecastedWriteLoad(writeIndex); - assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); - assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(14.4))); + assertThat(forecastedWriteLoad.isPresent(), is(true)); + assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(14.4))); } public void testForecastedWriteLoadIsOverriddenBySetting() { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); final AtomicBoolean hasValidLicense = new AtomicBoolean(true); - final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, maxIndexAge); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, maxIndexAge); final Metadata.Builder metadataBuilder = Metadata.builder(); final String dataStreamName = "logs-es"; @@ -173,15 +190,15 @@ public void testForecastedWriteLoadIsOverriddenBySetting() { final IndexMetadata writeIndex = updatedMetadataBuilder.getSafe(dataStream.getWriteIndex()); - final OptionalDouble forecastedWriteLoadForShard = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + final OptionalDouble forecastedWriteLoad = writeLoadForecaster.getForecastedWriteLoad(writeIndex); - assertThat(forecastedWriteLoadForShard.isPresent(), is(true)); - assertThat(forecastedWriteLoadForShard.getAsDouble(), is(equalTo(0.6))); + assertThat(forecastedWriteLoad.isPresent(), is(true)); + assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(0.6))); hasValidLicense.set(false); - final OptionalDouble forecastedWriteLoadForShardAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); - assertThat(forecastedWriteLoadForShardAfterLicenseChange.isPresent(), is(false)); + final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); + assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(false)); } public void testWriteLoadForecast() { @@ -271,7 +288,7 @@ public void testWriteLoadForecast() { public void testGetIndicesWithinMaxAgeRange() { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); - final LicensedWriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, maxIndexAge); + final LicensedWriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, threadPool, maxIndexAge); final Metadata.Builder metadataBuilder = Metadata.builder(); final int numberOfBackingIndicesOlderThanMinAge = randomIntBetween(0, 10); From 7d1acfaff31404f0ce4871ef28f7c0a46d42e86f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Mon, 14 Nov 2022 11:47:04 +0100 Subject: [PATCH 08/10] Delete changelog --- docs/changelog/91425.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/91425.yaml diff --git a/docs/changelog/91425.yaml b/docs/changelog/91425.yaml deleted file mode 100644 index 5095b2479e745..0000000000000 --- a/docs/changelog/91425.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 91425 -summary: "[WIP] Forecast normalized write load during rollovers" -area: Allocation -type: enhancement -issues: [] From e244c6d74b7714b21c8e4dab51dd86c1f3aa749a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Mon, 14 Nov 2022 11:59:43 +0100 Subject: [PATCH 09/10] Renaming --- server/src/main/java/org/elasticsearch/node/Node.java | 2 +- .../src/main/java/org/elasticsearch/plugins/ClusterPlugin.java | 2 +- .../xpack/writeloadforecaster/WriteLoadForecasterPlugin.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 44ad681a3f4f4..f8cabf6ccaf72 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -1244,7 +1244,7 @@ private RecoveryPlannerService getRecoveryPlannerService( private WriteLoadForecaster getWriteLoadForecaster(ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings) { final List clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class); final List writeLoadForecasters = clusterPlugins.stream() - .flatMap(clusterPlugin -> clusterPlugin.createWriteLoadHandler(threadPool, settings, clusterSettings).stream()) + .flatMap(clusterPlugin -> clusterPlugin.createWriteLoadForecasters(threadPool, settings, clusterSettings).stream()) .toList(); if (writeLoadForecasters.isEmpty()) { diff --git a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java index 8e0d23912d935..3ca862d887313 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java @@ -60,7 +60,7 @@ default Map getExistingShardsAllocators() { return Collections.emptyMap(); } - default Collection createWriteLoadHandler( + default Collection createWriteLoadForecasters( ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java index 99eca1895cd54..2272c1258ee3b 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterPlugin.java @@ -50,7 +50,7 @@ public List> getSettings() { } @Override - public Collection createWriteLoadHandler( + public Collection createWriteLoadForecasters( ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings From e01543b5064df79864df73426ed7ada6463669d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Mon, 14 Nov 2022 14:02:59 +0100 Subject: [PATCH 10/10] Update docs/changelog/91425.yaml --- docs/changelog/91425.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/91425.yaml diff --git a/docs/changelog/91425.yaml b/docs/changelog/91425.yaml new file mode 100644 index 0000000000000..3200fcc2655f2 --- /dev/null +++ b/docs/changelog/91425.yaml @@ -0,0 +1,5 @@ +pr: 91425 +summary: Forecast write load during rollovers +area: Allocation +type: enhancement +issues: []