From 527003d223530000ee550372865e9ea8b304ae1e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Fri, 4 Nov 2022 09:58:34 +0100 Subject: [PATCH 01/17] Downsampling transport service disruption (#90916) Test downsampling under two different failure scenarios. --- .../DownsampleTransportFailureTests.java | 378 ++++++++++++++++++ 1 file changed, 378 insertions(+) create mode 100644 x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleTransportFailureTests.java diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleTransportFailureTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleTransportFailureTests.java new file mode 100644 index 0000000000000..5395f1d557a8e --- /dev/null +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleTransportFailureTests.java @@ -0,0 +1,378 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.downsample.DownsampleAction; +import org.elasticsearch.xpack.core.downsample.DownsampleConfig; +import org.elasticsearch.xpack.core.downsample.RollupIndexerAction; +import org.elasticsearch.xpack.rollup.Rollup; +import org.junit.Before; + +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 2, numClientNodes = 1, supportsDedicatedMasters = false) +public class DownsampleTransportFailureTests extends ESIntegTestCase { + + public static final String ROLLUP_INDEXER_SHARD_ACTION = RollupIndexerAction.NAME + "[s]"; + + private static class TestClusterHelper { + private final InternalTestCluster cluster; + private final String coordinator; + private final String worker; + + TestClusterHelper(final InternalTestCluster cluster) { + this.cluster = cluster; + this.coordinator = randomCoordinator(); + this.worker = randomWorker(); + } + + private String randomCoordinator() { + return randomFrom(nonMasterNodes()); + } + + private String randomWorker() { + assert this.coordinator != null; + return randomFrom( + Arrays.stream(cluster.getNodeNames()) + .filter( + nodeName -> this.cluster.getMasterName().equals(nodeName) == false && this.coordinator.equals(nodeName) == false + ) + .collect(Collectors.toSet()) + ); + } + + private Collection nonMasterNodes() { + return Arrays.stream(cluster.getNodeNames()) + .filter(nodeName -> this.cluster.getMasterName().equals(nodeName) == false) + .collect(Collectors.toSet()); + } + + public int size() { + return this.cluster.size(); + } + + public String masterName() { + return cluster.getMasterName(); + } + + public Client coordinatorClient() { + assert this.coordinator != null; + return client(this.coordinator); + } + + public Client masterClient() { + return client(this.cluster.getMasterName()); + } + + public MockTransportService masterMockTransportService() { + return (MockTransportService) internalCluster().getInstance(TransportService.class, internalCluster().getMasterName()); + } + + public MockTransportService coordinatorMockTransportService() { + assert this.coordinator != null; + return (MockTransportService) internalCluster().getInstance(TransportService.class, this.coordinator); + } + + public List allMockTransportServices() { + return Arrays.stream(cluster.getNodeNames()) + .map(nodeName -> (MockTransportService) internalCluster().getInstance(TransportService.class, nodeName)) + .collect(Collectors.toList()); + } + + public String coordinatorName() { + assert this.coordinator != null; + return this.coordinator; + } + + public String workerName() { + assert this.worker != null; + return this.worker; + } + } + + private static final int DOWNSAMPLE_ACTION_TIMEOUT_MILLIS = 10_000; + private static final String SOURCE_INDEX_NAME = "source"; + private static final String TARGET_INDEX_NAME = "target"; + private long startTime; + private long endTime; + private TestClusterHelper testCluster; + + private final List DOCUMENTS = new ArrayList<>( + List.of( + "{\"@timestamp\": \"2020-09-09T18:03:00\",\"dim1\": \"dim1\",\"dim2\": \"401\",\"gauge\": \"100\",\"counter\": \"100\"}", + "{\"@timestamp\": \"2020-09-09T18:04:00\",\"dim1\": \"dim1\",\"dim2\": \"402\",\"gauge\": \"101\",\"counter\": \"101\"}", + "{\"@timestamp\": \"2020-09-09T18:05:00\",\"dim1\": \"dim1\",\"dim2\": \"403\",\"gauge\": \"102\",\"counter\": \"102\"}", + "{\"@timestamp\": \"2020-09-09T18:06:00\",\"dim1\": \"dim1\",\"dim2\": \"404\",\"gauge\": \"101\",\"counter\": \"103\"}", + "{\"@timestamp\": \"2020-09-09T18:07:00\",\"dim1\": \"dim1\",\"dim2\": \"405\",\"gauge\": \"103\",\"counter\": \"104\"}", + "{\"@timestamp\": \"2020-09-09T18:08:00\",\"dim1\": \"dim1\",\"dim2\": \"406\",\"gauge\": \"110\",\"counter\": \"105\"}", + "{\"@timestamp\": \"2020-09-09T18:09:00\",\"dim1\": \"dim1\",\"dim2\": \"407\",\"gauge\": \"112\",\"counter\": \"106\"}", + "{\"@timestamp\": \"2020-09-09T18:10:00\",\"dim1\": \"dim1\",\"dim2\": \"408\",\"gauge\": \"111\",\"counter\": \"107\"}", + "{\"@timestamp\": \"2020-09-09T18:11:00\",\"dim1\": \"dim1\",\"dim2\": \"409\",\"gauge\": \"105\",\"counter\": \"108\"}", + "{\"@timestamp\": \"2020-09-09T18:12:00\",\"dim1\": \"dim1\",\"dim2\": \"410\",\"gauge\": \"106\",\"counter\": \"109\"}", + "{\"@timestamp\": \"2020-09-09T18:13:00\",\"dim1\": \"dim1\",\"dim2\": \"411\",\"gauge\": \"107\",\"counter\": \"110\"}", + "{\"@timestamp\": \"2020-09-09T18:14:00\",\"dim1\": \"dim1\",\"dim2\": \"412\",\"gauge\": \"104\",\"counter\": \"111\"}" + ) + ); + + @Override + protected Collection> nodePlugins() { + return List.of(LocalStateCompositeXPackPlugin.class, Rollup.class, AggregateMetricMapperPlugin.class); + } + + @Override + protected Collection> getMockPlugins() { + return List.of(MockTransportService.TestPlugin.class, TestSeedPlugin.class); + } + + @Before + public void setup() throws IOException, ExecutionException, InterruptedException { + startTime = LocalDateTime.parse("2020-09-09T18:00:00").atZone(ZoneId.of("UTC")).toInstant().toEpochMilli(); + endTime = LocalDateTime.parse("2020-09-09T18:59:00").atZone(ZoneId.of("UTC")).toInstant().toEpochMilli(); + testCluster = new TestClusterHelper(internalCluster()); + assert testCluster.size() == 3; + ensureStableCluster(internalCluster().size()); + createTimeSeriesIndex(SOURCE_INDEX_NAME); + ensureGreen(SOURCE_INDEX_NAME); + indexDocuments(SOURCE_INDEX_NAME, DOCUMENTS); + blockIndexWrites(SOURCE_INDEX_NAME); + + logger.info( + "Cluster size {}, master node {}, coordinator node {}, worker node {}}", + testCluster.size(), + testCluster.masterName(), + testCluster.coordinatorName(), + testCluster.workerName() + ); + } + + @Override + public Settings indexSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("dim1")) + .put( + IndexSettings.TIME_SERIES_START_TIME.getKey(), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(Instant.ofEpochMilli(startTime).toEpochMilli()) + ) + .put( + IndexSettings.TIME_SERIES_END_TIME.getKey(), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(Instant.ofEpochMilli(endTime).toEpochMilli()) + ) + .build(); + } + + public XContentBuilder indexMapping() throws IOException { + final XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties"); + mapping.startObject("@timestamp").field("type", "date").endObject(); + mapping.startObject("dim1").field("type", "keyword").field("time_series_dimension", true).endObject(); + mapping.startObject("dim2").field("type", "long").field("time_series_dimension", true).endObject(); + mapping.startObject("gauge").field("type", "long").field("time_series_metric", "gauge").endObject(); + mapping.startObject("counter").field("type", "double").field("time_series_metric", "counter").endObject(); + mapping.endObject().endObject().endObject(); + return mapping; + } + + public void indexDocuments(final String indexName, final List documentsJson) { + final BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + documentsJson.forEach(document -> bulkRequestBuilder.add(new IndexRequest(indexName).source(document, XContentType.JSON))); + assertFalse(bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get().hasFailures()); + } + + public void blockIndexWrites(final String indexName) throws ExecutionException, InterruptedException { + final Settings blockWritesSetting = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build(); + assertTrue( + client().admin().indices().updateSettings(new UpdateSettingsRequest(blockWritesSetting, indexName)).get().isAcknowledged() + ); + } + + private void createTimeSeriesIndex(final String indexName) throws IOException { + assertTrue(prepareCreate(indexName).setMapping(indexMapping()).get().isShardsAcknowledged()); + } + + private void assertDownsampleFailure(final String nodeName) { + assertIndexExists(nodeName, SOURCE_INDEX_NAME); + assertDocumentsExist(nodeName, SOURCE_INDEX_NAME); + assertIndexDoesNotExist(nodeName, TARGET_INDEX_NAME); + } + + private void assertDocumentsExist(final String nodeName, final String indexName) { + final SearchResponse searchResponse = client(nodeName).prepareSearch(indexName) + .setQuery(new MatchAllQueryBuilder()) + .setTrackTotalHitsUpTo(Integer.MAX_VALUE) + .setSize(DOCUMENTS.size()) + .get(); + assertEquals(DOCUMENTS.size(), searchResponse.getHits().getHits().length); + } + + private void assertIndexExists(final String nodeName, final String indexName) { + final GetIndexResponse getIndexResponse = client(nodeName).admin() + .indices() + .prepareGetIndex() + .addIndices(indexName) + .addFeatures(GetIndexRequest.Feature.values()) + .get(); + assertEquals(List.of(indexName), Arrays.stream(getIndexResponse.indices()).toList()); + } + + private void assertIndexDoesNotExist(final String nodeName, final String indexName) { + final IndexNotFoundException targetIndexNotFoundException = expectThrows( + IndexNotFoundException.class, + "Index [" + indexName + "] was not deleted", + () -> client(nodeName).admin() + .indices() + .prepareGetIndex() + .addIndices(indexName) + .addFeatures(GetIndexRequest.Feature.values()) + .get() + ); + assertEquals("no such index [" + indexName + "]", targetIndexNotFoundException.getMessage()); + } + + public void testNoDisruption() { + // GIVEN + + final DownsampleAction.Request downsampleRequest = new DownsampleAction.Request( + SOURCE_INDEX_NAME, + TARGET_INDEX_NAME, + new DownsampleConfig(DateHistogramInterval.MINUTE) + ); + + // WHEN nothing happens + + // THEN + final AcknowledgedResponse downsampleResponse = testCluster.masterClient() + .execute(DownsampleAction.INSTANCE, downsampleRequest) + .actionGet(TimeValue.timeValueMillis(DOWNSAMPLE_ACTION_TIMEOUT_MILLIS)); + assertTrue(downsampleResponse.isAcknowledged()); + + assertIndexExists(testCluster.coordinatorName(), SOURCE_INDEX_NAME); + assertDocumentsExist(testCluster.coordinatorName(), SOURCE_INDEX_NAME); + assertIndexExists(testCluster.coordinatorName(), TARGET_INDEX_NAME); + // NOTE: the target downsample index `fixed_interval` matches the source index @timestamp interval. + // As a result, the target index includes the same number of documents of the source index. + assertDocumentsExist(testCluster.coordinatorName(), TARGET_INDEX_NAME); + ensureStableCluster(internalCluster().size()); + } + + public void testDownsampleActionExceptionDisruption() { + // GIVEN + final MockTransportService coordinator = testCluster.coordinatorMockTransportService(); + final DownsampleAction.Request downsampleRequest = new DownsampleAction.Request( + SOURCE_INDEX_NAME, + TARGET_INDEX_NAME, + new DownsampleConfig(DateHistogramInterval.HOUR) + ); + + // WHEN (disruption) + testCluster.allMockTransportServices() + .forEach( + mockTransportService -> coordinator.addSendBehavior( + mockTransportService, + (connection, requestId, action, request, options) -> { + if (DownsampleAction.NAME.equals(action)) { + logger.info("Simulated disruption: node [" + connection.getNode().getName() + "] action [" + action + "]"); + throw new ElasticsearchException( + "Simulated disruption: node [" + connection.getNode().getName() + "] action [" + action + "]" + ); + } + connection.sendRequest(requestId, action, request, options); + } + ) + ); + + // THEN + expectThrows( + ElasticsearchException.class, + () -> testCluster.coordinatorClient() + .execute(DownsampleAction.INSTANCE, downsampleRequest) + .actionGet(TimeValue.timeValueMillis(DOWNSAMPLE_ACTION_TIMEOUT_MILLIS)) + ); + + coordinator.clearAllRules(); + ensureStableCluster(testCluster.size()); + assertDownsampleFailure(testCluster.coordinatorName()); + } + + public void testRollupIndexerActionExceptionDisruption() { + // GIVEN + final MockTransportService master = testCluster.masterMockTransportService(); + final DownsampleAction.Request downsampleRequest = new DownsampleAction.Request( + SOURCE_INDEX_NAME, + TARGET_INDEX_NAME, + new DownsampleConfig(DateHistogramInterval.HOUR) + ); + + // WHEN (disruption) + testCluster.allMockTransportServices() + .forEach( + mockTransportService -> master.addSendBehavior(mockTransportService, (connection, requestId, action, request, options) -> { + if (ROLLUP_INDEXER_SHARD_ACTION.equals(action)) { + logger.info("Simulated disruption: node [" + connection.getNode().getName() + "] action [" + action + "]"); + throw new ElasticsearchException( + "Simulated disruption: node [" + connection.getNode().getName() + "] action [" + action + "]" + ); + } + connection.sendRequest(requestId, action, request, options); + }) + ); + + // THEN + expectThrows( + ElasticsearchException.class, + () -> testCluster.coordinatorClient() + .execute(DownsampleAction.INSTANCE, downsampleRequest) + .actionGet(TimeValue.timeValueMillis(DOWNSAMPLE_ACTION_TIMEOUT_MILLIS)) + ); + + master.clearAllRules(); + ensureStableCluster(testCluster.size()); + assertDownsampleFailure(testCluster.coordinatorName()); + } +} From b5743811f11a72a63b3585122c895a6f8f0f2d27 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 4 Nov 2022 12:01:11 +0200 Subject: [PATCH 02/17] Refactor RBACEngine authorize index action for resolved indices (#91180) Refactor index action authorization to a single code path, in both cases where the request allows remote indices or not. --- .../security/authz/AuthorizationEngine.java | 4 + .../ProfileCancellationIntegTests.java | 3 +- .../xpack/security/authz/RBACEngine.java | 118 ++++++------------ 3 files changed, 46 insertions(+), 79 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java index 79116a5b624a8..4c5bd39cc7748 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java @@ -551,6 +551,10 @@ public static AuthorizationResult deny() { */ class IndexAuthorizationResult extends AuthorizationResult { + public static final IndexAuthorizationResult DENIED = new IndexAuthorizationResult(IndicesAccessControl.DENIED); + public static final IndexAuthorizationResult EMPTY = new IndexAuthorizationResult(null); + public static final IndexAuthorizationResult ALLOW_NO_INDICES = new IndexAuthorizationResult(IndicesAccessControl.ALLOW_NO_INDICES); + private final IndicesAccessControl indicesAccessControl; public IndexAuthorizationResult(IndicesAccessControl indicesAccessControl) { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java index 860e7dfe1877a..58be2dffcc3d7 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java @@ -41,7 +41,6 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; -import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; import org.elasticsearch.xpack.security.LocalStateSecurity; @@ -410,7 +409,7 @@ public void authorizeIndexAction( Map aliasOrIndexLookup, ActionListener listener ) { - listener.onResponse(new IndexAuthorizationResult(IndicesAccessControl.ALLOW_NO_INDICES)); + listener.onResponse(IndexAuthorizationResult.ALLOW_NO_INDICES); } @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index fcfc1cb775124..1d3c3f7cfb1d9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -278,14 +278,17 @@ public void authorizeIndexAction( ) { final String action = requestInfo.getAction(); final TransportRequest request = requestInfo.getRequest(); + final Role role; + try { + role = ensureRBAC(authorizationInfo).getRole(); + } catch (Exception e) { + listener.onFailure(e); + return; + } if (TransportActionProxy.isProxyAction(action) || shouldAuthorizeIndexActionNameOnly(action, request)) { // we've already validated that the request is a proxy request so we can skip that but we still // need to validate that the action is allowed and then move on - try { - listener.onResponse(authorizeIndexActionName(action, authorizationInfo, null)); - } catch (Exception e) { - listener.onFailure(e); - } + listener.onResponse(role.checkIndicesAction(action) ? IndexAuthorizationResult.EMPTY : IndexAuthorizationResult.DENIED); } else if (request instanceof IndicesRequest == false) { if (isScrollRelatedAction(action)) { // scroll is special @@ -305,9 +308,11 @@ public void authorizeIndexAction( if (SearchScrollAction.NAME.equals(action)) { ActionRunnable.supply(ActionListener.wrap(parsedScrollId -> { if (parsedScrollId.hasLocalIndices()) { - listener.onResponse(authorizeIndexActionName(action, authorizationInfo, null)); + listener.onResponse( + role.checkIndicesAction(action) ? IndexAuthorizationResult.EMPTY : IndexAuthorizationResult.DENIED + ); } else { - listener.onResponse(new IndexAuthorizationResult(null)); + listener.onResponse(IndexAuthorizationResult.EMPTY); } }, listener::onFailure), ((SearchScrollRequest) request)::parseScrollId).run(); } else { @@ -319,21 +324,21 @@ public void authorizeIndexAction( // The DLS/FLS permissions are used inside the {@code DirectoryReader} that {@code SecurityIndexReaderWrapper} // built while handling the initial search request. In addition, for consistency, the DLS/FLS permissions from // the originating search request are attached to the thread context upon validating the scroll. - listener.onResponse(new IndexAuthorizationResult(null)); + listener.onResponse(IndexAuthorizationResult.EMPTY); } } else if (isAsyncRelatedAction(action)) { if (SubmitAsyncSearchAction.NAME.equals(action)) { // authorize submit async search but don't fill in the DLS/FLS permissions // the `null` IndicesAccessControl parameter indicates that this action has *not* determined // which DLS/FLS controls should be applied to this action - listener.onResponse(new IndexAuthorizationResult(null)); + listener.onResponse(IndexAuthorizationResult.EMPTY); } else { // async-search actions other than submit have a custom security layer that checks if the current user is // the same as the user that submitted the original request so no additional checks are needed here. - listener.onResponse(new IndexAuthorizationResult(IndicesAccessControl.ALLOW_NO_INDICES)); + listener.onResponse(IndexAuthorizationResult.ALLOW_NO_INDICES); } } else if (action.equals(ClosePointInTimeAction.NAME)) { - listener.onResponse(new IndexAuthorizationResult(IndicesAccessControl.ALLOW_NO_INDICES)); + listener.onResponse(IndexAuthorizationResult.ALLOW_NO_INDICES); } else { assert false : "only scroll and async-search related requests are known indices api that don't " @@ -347,74 +352,39 @@ public void authorizeIndexAction( } } else if (isChildActionAuthorizedByParent(requestInfo, authorizationInfo)) { listener.onResponse(new IndexAuthorizationResult(requestInfo.getOriginatingAuthorizationContext().getIndicesAccessControl())); - } else if (request instanceof IndicesRequest.Replaceable replaceable && replaceable.allowsRemoteIndices()) { - // remote indices are allowed + } else if (allowsRemoteIndices(request) || role.checkIndicesAction(action)) { indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> { assert resolvedIndices.isEmpty() == false : "every indices request needs to have its indices set thus the resolved indices must not be empty"; // all wildcard expressions have been resolved and only the security plugin could have set '-*' here. // '-*' matches no indices so we allow the request to go through, which will yield an empty response if (resolvedIndices.isNoIndicesPlaceholder()) { - // check action name - listener.onResponse(authorizeIndexActionName(action, authorizationInfo, IndicesAccessControl.ALLOW_NO_INDICES)); + if (allowsRemoteIndices(request) && role.checkIndicesAction(action) == false) { + listener.onResponse(IndexAuthorizationResult.DENIED); + } else { + listener.onResponse(IndexAuthorizationResult.ALLOW_NO_INDICES); + } } else { assert resolvedIndices.getLocal().stream().noneMatch(Regex::isSimpleMatchPattern) - || replaceable.indicesOptions().expandWildcardExpressions() == false + || ((IndicesRequest) request).indicesOptions().expandWildcardExpressions() == false || (request instanceof AliasesRequest aliasesRequest && aliasesRequest.expandAliasesWildcards() == false) + || (request instanceof IndicesAliasesRequest indicesAliasesRequest + && false == indicesAliasesRequest.getAliasActions() + .stream() + .allMatch(IndicesAliasesRequest.AliasActions::expandAliasesWildcards)) : "expanded wildcards for local indices OR the request should not expand wildcards at all"; - listener.onResponse( - buildIndicesAccessControl( - action, - authorizationInfo, - Sets.newHashSet(resolvedIndices.getLocal()), - aliasOrIndexLookup - ) - ); + listener.onResponse(buildIndicesAccessControl(action, role, resolvedIndices, aliasOrIndexLookup)); } }, listener::onFailure)); } else { - try { - final IndexAuthorizationResult indexAuthorizationResult = authorizeIndexActionName( - action, - authorizationInfo, - IndicesAccessControl.ALLOW_NO_INDICES - ); - if (indexAuthorizationResult.isGranted()) { - indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> { - assert resolvedIndices.isEmpty() == false - : "every indices request needs to have its indices set thus the resolved indices must not be empty"; - // all wildcard expressions have been resolved and only the security plugin could have set '-*' here. - // '-*' matches no indices so we allow the request to go through, which will yield an empty response - if (resolvedIndices.isNoIndicesPlaceholder()) { - listener.onResponse(new IndexAuthorizationResult(IndicesAccessControl.ALLOW_NO_INDICES)); - } else { - assert resolvedIndices.getLocal().stream().noneMatch(Regex::isSimpleMatchPattern) - || ((IndicesRequest) request).indicesOptions().expandWildcardExpressions() == false - || (request instanceof AliasesRequest aliasesRequest && aliasesRequest.expandAliasesWildcards() == false) - || (request instanceof IndicesAliasesRequest indicesAliasesRequest - && false == indicesAliasesRequest.getAliasActions() - .stream() - .allMatch(IndicesAliasesRequest.AliasActions::expandAliasesWildcards)) - : "expanded wildcards for local indices OR the request should not expand wildcards at all"; - listener.onResponse( - buildIndicesAccessControl( - action, - authorizationInfo, - Sets.newHashSet(resolvedIndices.getLocal()), - aliasOrIndexLookup - ) - ); - } - }, listener::onFailure)); - } else { - listener.onResponse(indexAuthorizationResult); - } - } catch (Exception e) { - listener.onFailure(e); - } + listener.onResponse(IndexAuthorizationResult.DENIED); } } + private static boolean allowsRemoteIndices(TransportRequest transportRequest) { + return transportRequest instanceof IndicesRequest.Replaceable replaceable && replaceable.allowsRemoteIndices(); + } + private static boolean isChildActionAuthorizedByParent(RequestInfo requestInfo, AuthorizationInfo authorizationInfo) { final AuthorizationContext parent = requestInfo.getOriginatingAuthorizationContext(); if (parent == null) { @@ -479,16 +449,6 @@ private static boolean isChildActionAuthorizedByParent(RequestInfo requestInfo, return Arrays.stream(indices).allMatch(indicesAccessControl::hasIndexPermissions); } - private static IndexAuthorizationResult authorizeIndexActionName( - String action, - AuthorizationInfo authorizationInfo, - IndicesAccessControl grantedValue - ) { - final Role role = ensureRBAC(authorizationInfo).getRole(); - return new IndexAuthorizationResult(role.checkIndicesAction(action) ? grantedValue : IndicesAccessControl.DENIED); - - } - @Override public void loadAuthorizedIndices( RequestInfo requestInfo, @@ -834,12 +794,16 @@ static Set resolveAuthorizedIndicesFromRole( private IndexAuthorizationResult buildIndicesAccessControl( String action, - AuthorizationInfo authorizationInfo, - Set indices, + Role role, + ResolvedIndices resolvedIndices, Map aliasAndIndexLookup ) { - final Role role = ensureRBAC(authorizationInfo).getRole(); - final IndicesAccessControl accessControl = role.authorize(action, indices, aliasAndIndexLookup, fieldPermissionsCache); + final IndicesAccessControl accessControl = role.authorize( + action, + Sets.newHashSet(resolvedIndices.getLocal()), + aliasAndIndexLookup, + fieldPermissionsCache + ); return new IndexAuthorizationResult(accessControl); } From ac4dba7f2f38f248d388cc99e42769c5617be9bc Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:02:27 +0100 Subject: [PATCH 03/17] Test data stream downsample (#91141) The data stream is created so that it includes a downsampled index and a normal write index. The date histogram query uses a fixed_interval value that is smaller than the downsample fixed_interval aggregation interval. --- .../downsample/DownsampleDataStreamTests.java | 248 ++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java new file mode 100644 index 0000000000000..e81cf9c6d4acd --- /dev/null +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java @@ -0,0 +1,248 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamAction; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.downsample.DownsampleAction; +import org.elasticsearch.xpack.core.downsample.DownsampleConfig; +import org.elasticsearch.xpack.rollup.Rollup; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.ExecutionException; + +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.hamcrest.Matchers.equalTo; + +public class DownsampleDataStreamTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return List.of(Rollup.class, DataStreamsPlugin.class); + } + + public void testDataStreamDownsample() throws ExecutionException, InterruptedException, IOException { + // GIVEN + final String dataStreamName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("1", List.of(dataStreamName)); + client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet(); + indexDocs(dataStreamName, 10, Instant.now().toEpochMilli()); + final RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).get(); + // NOTE: here we calculate a delay to index documents because the next data stream write index is created with a start time of + // (about) two hours in the future. As a result, we need to have documents whose @timestamp is in the future to avoid documents + // being indexed in the old data stream backing index. + final String newIndexStartTime = client().admin() + .indices() + .prepareGetSettings(rolloverResponse.getNewIndex()) + .get() + .getSetting(rolloverResponse.getNewIndex(), IndexSettings.TIME_SERIES_START_TIME.getKey()); + indexDocs(dataStreamName, 10, Instant.parse(newIndexStartTime).toEpochMilli()); + client().admin() + .indices() + .updateSettings( + new UpdateSettingsRequest().indices(rolloverResponse.getOldIndex()) + .settings(Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build()) + ) + .actionGet(); + + // WHEN (simulate downsampling as done by an ILM action) + final String downsampleTargetIndex = DataStream.BACKING_INDEX_PREFIX + dataStreamName + "-downsample-1h"; + final DownsampleAction.Request downsampleRequest = new DownsampleAction.Request( + rolloverResponse.getOldIndex(), + downsampleTargetIndex, + new DownsampleConfig(DateHistogramInterval.HOUR) + ); + final AcknowledgedResponse downsampleResponse = client().admin() + .indices() + .execute(DownsampleAction.INSTANCE, downsampleRequest) + .actionGet(); + + /* + * Force an index update to avoid failing with "Index updates are expected as index settings version has changed", + * due to a possible bug while checking settings versions and actual settings/metadata changes. + * See {@link IndexSettings#updateIndexMetadata}. + */ + client().admin() + .indices() + .updateSettings( + new UpdateSettingsRequest().indices(downsampleTargetIndex) + .settings(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, false).build()) + ) + .actionGet(); + + final ModifyDataStreamsAction.Request modifyDataStreamRequest = new ModifyDataStreamsAction.Request( + List.of( + DataStreamAction.removeBackingIndex(dataStreamName, rolloverResponse.getOldIndex()), + DataStreamAction.addBackingIndex(dataStreamName, downsampleTargetIndex) + ) + ); + client().execute(ModifyDataStreamsAction.INSTANCE, modifyDataStreamRequest).actionGet(); + + // THEN + assertThat(downsampleResponse.isAcknowledged(), equalTo(true)); + final GetDataStreamAction.Response getDataStreamActionResponse = client().admin() + .indices() + .execute(GetDataStreamAction.INSTANCE, new GetDataStreamAction.Request(new String[] { dataStreamName })) + .actionGet(); + assertThat(getDataStreamActionResponse.getDataStreams().get(0).getDataStream().getIndices().size(), equalTo(2)); + final List backingIndices = getDataStreamActionResponse.getDataStreams() + .get(0) + .getDataStream() + .getIndices() + .stream() + .map(Index::getName) + .toList(); + assertThat(backingIndices, Matchers.contains(downsampleTargetIndex, rolloverResponse.getNewIndex())); + + final SearchRequest searchRequest = new SearchRequest().indices(dataStreamName) + .source( + new SearchSourceBuilder().size(20) + .query(new MatchAllQueryBuilder()) + .sort(SortBuilders.fieldSort("@timestamp").order(SortOrder.DESC)) + .aggregation( + new DateHistogramAggregationBuilder("dateHistogram").field("@timestamp").fixedInterval(DateHistogramInterval.MINUTE) + ) + ); + final SearchResponse searchResponse = client().search(searchRequest).actionGet(); + Arrays.stream(searchResponse.getHits().getHits()) + .limit(10) + .forEach(hit -> assertThat(hit.getIndex(), equalTo(rolloverResponse.getNewIndex()))); + assertThat(searchResponse.getHits().getHits()[10].getIndex(), equalTo(downsampleTargetIndex)); + final InternalDateHistogram dateHistogram = searchResponse.getAggregations().get("dateHistogram"); + // NOTE: due to unpredictable values for the @timestamp field we don't know how many buckets we have in the + // date histogram. We know, anyway, that we will have 10 documents in the first two buckets, 10 documents in the last two buckets. + // The actual number of documents on each of the first two and last two buckets depends on the timestamp value generated when + // indexing + // documents, which might cross the minute boundary of the fixed_interval date histogram aggregation. + // Then we check there is a variable number of intermediate buckets with exactly 0 documents. This is a result of the way + // downsampling + // deals with a fixed interval granularity that is larger than the date histogram fixed interval (1 minute (date histogram + // fixed_interval) + // < 1 hour (downsample fixed_interval)). + final int totalBuckets = dateHistogram.getBuckets().size(); + assertThat(dateHistogram.getBuckets().get(0).getDocCount() + dateHistogram.getBuckets().get(1).getDocCount(), equalTo(10L)); + dateHistogram.getBuckets() + .stream() + .skip(2) + .limit(totalBuckets - 3) + .map(InternalDateHistogram.Bucket::getDocCount) + .toList() + .forEach(docCount -> assertThat(docCount, equalTo(0L))); + assertThat( + dateHistogram.getBuckets().get(totalBuckets - 2).getDocCount() + dateHistogram.getBuckets().get(totalBuckets - 1).getDocCount(), + equalTo(10L) + ); + } + + private void putComposableIndexTemplate(final String id, final List patterns) throws IOException { + final PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(id); + final Template template = new Template( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("routing_field")) + .build(), + new CompressedXContent(""" + { + "properties": { + "@timestamp" : { + "type": "date" + }, + "routing_field": { + "type": "keyword", + "time_series_dimension": true + }, + "counter": { + "type": "long", + "time_series_metric": "counter" + } + } + } + """), + null + ); + request.indexTemplate( + new ComposableIndexTemplate(patterns, template, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null) + ); + client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); + } + + private void indexDocs(final String dataStream, int numDocs, long startTime) { + final BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < numDocs; i++) { + final String timestamp = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(startTime + i); + bulkRequest.add( + new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + .source( + String.format( + Locale.ROOT, + "{\"%s\":\"%s\",\"%s\":\"%s\",\"%s\":\"%s\"}", + DEFAULT_TIMESTAMP_FIELD, + timestamp, + "routing_field", + 0, + "counter", + i + 1 + ), + XContentType.JSON + ) + ); + } + final BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + final BulkItemResponse[] items = bulkResponse.getItems(); + assertThat(items.length, equalTo(numDocs)); + assertThat(bulkResponse.hasFailures(), equalTo(false)); + final RefreshResponse refreshResponse = client().admin().indices().refresh(new RefreshRequest(dataStream)).actionGet(); + assertThat(refreshResponse.getStatus().getStatus(), equalTo(RestStatus.OK.getStatus())); + } +} From cfc4e4725b36990a05cd5282357a7debaf955b8b Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 4 Nov 2022 11:53:00 +0100 Subject: [PATCH 04/17] [TEST] Expand tests for special field names in docs and mappings (#91043) We have some existing tests that verify that we throw errors when parsing certain special field names in documents. This commit expands them to test more edge cases, as well as to test the same scearios with subobjects:false as well as dynamic:runtime. Furthermore, additional tests and checks are added to verify that the behaviour is aligned between document parsing and mapping parsing, which is not the case in many scenarios. This commit does not aim at fixing the anomalies found, but only to surface them and have tests that can later be adapted once the different scenarios are fixed. --- .../index/mapper/DocumentParserTests.java | 177 +++++++++++++----- .../index/mapper/DynamicMappingTests.java | 9 +- .../index/mapper/MappingParserTests.java | 131 +++++++++++++ .../index/mapper/ObjectMapperTests.java | 43 ++--- .../index/mapper/MapperServiceTestCase.java | 9 + 5 files changed, 292 insertions(+), 77 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 95e43fcd3723a..7eb99a9817603 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -38,6 +38,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import java.util.function.Function; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; @@ -1806,49 +1807,119 @@ public void testDynamicDateDetectionEnabledWithNoSpecialCharacters() throws IOEx assertThat(dateMapper, instanceOf(DateFieldMapper.class)); } - public void testDynamicFieldsStartingAndEndingWithDot() throws Exception { - MapperService mapperService = createMapperService(mapping(b -> {})); - Exception e = expectThrows(MapperParsingException.class, () -> mapperService.documentMapper().parse(source(""" - {"top..foo.":{"a":1}} - """))); + private void dynamicTrueOrDynamicRuntimeTest(Consumer mapperServiceConsumer) throws Exception { + for (XContentBuilder xContentBuilder : new XContentBuilder[] { mapping(b -> {}), topMapping(b -> b.field("dynamic", "runtime")) }) { + mapperServiceConsumer.accept(createMapperService(xContentBuilder).documentMapper()); + } + } - assertThat(e.getCause().getMessage(), containsString("field name cannot contain only whitespace: ['top..foo.']")); + public void testDynamicFieldStartingWithDot() throws Exception { + dynamicTrueOrDynamicRuntimeTest(mapper -> { + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(""" + {".foo":1} + """))); + // TODO isn't this a misleading error? + assertThat(e.getCause().getMessage(), containsString("field name cannot contain only whitespace: ['.foo']")); + }); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/28948") + public void testDynamicFieldEndingWithDot() throws Exception { + dynamicTrueOrDynamicRuntimeTest(mapper -> { + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(""" + {"foo.":1} + """))); + // TODO possibly throw a clearer error? + assertThat(e.getCause().getMessage(), containsString("field name cannot contain only whitespace: ['foo.']")); + }); + } + + public void testDynamicDottedFieldWithTrailingDots() throws Exception { + dynamicTrueOrDynamicRuntimeTest(mapper -> { + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(""" + {"top..foo":1} + """))); + // TODO isn't this a misleading error? + assertThat(e.getCause().getMessage(), containsString("field name cannot contain only whitespace: ['top..foo']")); + }); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/28948") + public void testDynamicDottedFieldEndingWithDot() throws Exception { + dynamicTrueOrDynamicRuntimeTest(mapper -> { + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(""" + {"top.foo.":1} + """))); + // TODO possibly throw a clearer error? + assertThat(e.getCause().getMessage(), containsString("field name cannot contain only whitespace: ['top.foo.']")); + }); } - public void testDynamicFieldsEmptyName() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + public void testDynamicFieldsStartingAndEndingWithDot() throws Exception { + dynamicTrueOrDynamicRuntimeTest(mapper -> { + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(""" + {"top..foo.":1} + """))); + // TODO isn't this a misleading error? + assertThat(e.getCause().getMessage(), containsString("field name cannot contain only whitespace: ['top..foo.']")); + }); + } + + public void testDynamicDottedFieldWithTrailingWhitespace() throws Exception { + dynamicTrueOrDynamicRuntimeTest(mapper -> { + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(""" + {"top. .foo":1} + """))); + // TODO isn't this a misleading error? + assertThat( + e.getCause().getMessage(), + containsString("field name starting or ending with a [.] makes object resolution ambiguous: [top. .foo]") + ); + }); + } - Exception emptyFieldNameException = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> { - b.startArray("top"); - { - b.startObject(); + public void testDynamicFieldsEmptyName() throws Exception { + dynamicTrueOrDynamicRuntimeTest(mapper -> { + Exception exception = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> { + b.startArray("top"); { - b.startObject("aoeu").field("a", 1).field(" ", 2).endObject(); + b.startObject(); + { + b.startObject("aoeu").field("a", 1).field(" ", 2).endObject(); + } + b.endObject(); } - b.endObject(); - } - b.endArray(); - }))); + b.endArray(); + }))); - assertThat(emptyFieldNameException.getMessage(), containsString("Field name cannot contain only whitespace: [top.aoeu. ]")); + assertThat(exception.getMessage(), containsString("Field name cannot contain only whitespace: [top.aoeu. ]")); + }); } public void testBlankFieldNames() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); - MapperParsingException err = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("", "foo")))); - assertThat(err.getCause(), notNullValue()); - assertThat(err.getCause().getMessage(), containsString("field name cannot be an empty string")); - - err = expectThrows( - MapperParsingException.class, - () -> mapper.parse(source(b -> b.startObject("foo").field("", "bar").endObject())) - ); - assertThat(err.getCause(), notNullValue()); - assertThat(err.getCause().getMessage(), containsString("field name cannot be an empty string")); + dynamicTrueOrDynamicRuntimeTest(mapper -> { + { + MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("", "foo")))); + assertThat(e.getCause(), notNullValue()); + assertThat(e.getCause().getMessage(), containsString("field name cannot be an empty string")); + } + { + MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field(" ", "foo")))); + assertThat(e.getMessage(), containsString("Field name cannot contain only whitespace: [ ]")); + } + { + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse(source(b -> b.startObject("foo").field("", "bar").endObject())) + ); + assertThat(e.getCause(), notNullValue()); + assertThat(e.getCause().getMessage(), containsString("field name cannot be an empty string")); + } + }); } public void testBlankFieldNamesSubobjectsFalse() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("subobjects", false))); + DocumentMapper mapper = createDocumentMapper(mappingNoSubobjects(b -> {})); { MapperParsingException err = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("", "foo")))); assertThat(err.getMessage(), containsString("Field name cannot contain only whitespace: []")); @@ -1860,20 +1931,42 @@ public void testBlankFieldNamesSubobjectsFalse() throws Exception { } public void testDotsOnlyFieldNames() throws Exception { - dotsOnlyFieldNames(createDocumentMapper(mapping(b -> {}))); + dynamicTrueOrDynamicRuntimeTest(mapper -> { + String[] fieldNames = { ".", "..", "..." }; + for (String fieldName : fieldNames) { + MapperParsingException err = expectThrows( + MapperParsingException.class, + () -> mapper.parse(source(b -> b.field(fieldName, "bar"))) + ); + assertThat(err.getCause(), notNullValue()); + assertThat(err.getCause().getMessage(), containsString("field name cannot contain only dots")); + } + }); } public void testDotsOnlyFieldNamesSubobjectsFalse() throws Exception { - dotsOnlyFieldNames(createDocumentMapper(topMapping(b -> b.field("subobjects", false)))); + DocumentMapper mapper = createDocumentMapper(mappingNoSubobjects(b -> {})); + String[] fieldNames = { ".", "..", "..." }; + for (String fieldName : fieldNames) { + MapperParsingException err = expectThrows( + MapperParsingException.class, + () -> mapper.parse(source(b -> b.field(fieldName, "bar"))) + ); + assertThat(err.getCause(), notNullValue()); + // TODO this is actually accepted in the mappings, shall we revert https://github.com/elastic/elasticsearch/pull/90950 ? + assertThat(err.getCause().getMessage(), containsString("field name cannot contain only dots")); + } } - private void dotsOnlyFieldNames(DocumentMapper mapper) { - MapperParsingException err = expectThrows( - MapperParsingException.class, - () -> mapper.parse(source(b -> b.field(randomFrom(".", "..", "..."), "bar"))) - ); - assertThat(err.getCause(), notNullValue()); - assertThat(err.getCause().getMessage(), containsString("field name cannot contain only dots")); + public void testDynamicFieldEdgeCaseNamesSubobjectsFalse() throws Exception { + // these combinations are not accepted by default, but they are when subobjects are disabled + MapperService mapperService = createMapperService(mappingNoSubobjects(b -> {})); + String[] fieldNames = new String[] { ".foo", "foo.", "top..foo", "top.foo.", "top..foo.", "top. .foo" }; + for (String fieldName : fieldNames) { + ParsedDocument doc = mapperService.documentMapper().parse(source("{\"" + fieldName + "\":1}")); + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); + assertNotNull(mapperService.fieldType(fieldName)); + } } public void testSubobjectsFalseWithInnerObject() throws Exception { @@ -1916,7 +2009,7 @@ public void testSubobjectsFalseWithInnerDottedObject() throws Exception { } public void testSubobjectsFalseRootWithInnerObject() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("subobjects", false))); + DocumentMapper mapper = createDocumentMapper(mappingNoSubobjects(xContentBuilder -> {})); MapperParsingException err = expectThrows(MapperParsingException.class, () -> mapper.parse(source(""" { "metrics": { @@ -1930,7 +2023,7 @@ public void testSubobjectsFalseRootWithInnerObject() throws Exception { } public void testSubobjectsFalseRoot() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("subobjects", false))); + DocumentMapper mapper = createDocumentMapper(mappingNoSubobjects(xContentBuilder -> {})); ParsedDocument doc = mapper.parse(source(""" { "metrics.service.time" : 10, @@ -2444,7 +2537,7 @@ public void testDeeplyNestedDocument() throws Exception { assertThat(mpe.getCause().getMessage(), containsString("Limit of mapping depth [20] has been exceeded due to object field")); // check that multiple-dotted field name underneath an object mapper with subobjects=false does not trigger this - DocumentMapper docMapper2 = createMapperService(topMapping(b -> { b.field("subobjects", false); })).documentMapper(); + DocumentMapper docMapper2 = createMapperService(mappingNoSubobjects(xContentBuilder -> {})).documentMapper(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < depth; i++) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index bdd31b90096f1..56bddfa164071 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -904,12 +904,9 @@ public void testArraysOfObjectsDynamicFalse() throws IOException { } public void testSubobjectsFalseRootDynamicUpdate() throws Exception { - MapperService mapperService = createMapperService(topMapping(b -> { - b.field("subobjects", false); - b.startObject("properties"); - b.startObject("host.name").field("type", "keyword").endObject(); - b.endObject(); - })); + MapperService mapperService = createMapperService( + mappingNoSubobjects(b -> b.startObject("host.name").field("type", "keyword").endObject()) + ); ParsedDocument doc = mapperService.documentMapper().parse(source(""" { "time" : 10, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 2e0c07940a562..06c66ac6463cb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -206,4 +206,135 @@ to get the wrong path (missing the first portion). assertEquals("location", geoPointFieldMapper.simpleName()); assertEquals("obj.source.geo.location", geoPointFieldMapper.mappedFieldType.name()); } + + public void testFieldStartingWithDot() throws Exception { + XContentBuilder builder = mapping(b -> b.startObject(".foo").field("type", "keyword").endObject()); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))) + ); + // TODO isn't this error misleading? + assertEquals("name cannot be empty string", iae.getMessage()); + } + + public void testFieldEndingWithDot() throws Exception { + XContentBuilder builder = mapping(b -> b.startObject("foo.").field("type", "keyword").endObject()); + Mapping mapping = createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); + // TODO this needs fixing as part of addressing https://github.com/elastic/elasticsearch/issues/28948 + assertNotNull(mapping.getRoot().mappers.get("foo")); + assertNull(mapping.getRoot().mappers.get("foo.")); + } + + public void testFieldTrailingDots() throws Exception { + XContentBuilder builder = mapping(b -> b.startObject("top..foo").field("type", "keyword").endObject()); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))) + ); + // TODO isn't this error misleading? + assertEquals("name cannot be empty string", iae.getMessage()); + } + + public void testDottedFieldEndingWithDot() throws Exception { + XContentBuilder builder = mapping(b -> b.startObject("foo.bar.").field("type", "keyword").endObject()); + Mapping mapping = createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); + // TODO this needs fixing as part of addressing https://github.com/elastic/elasticsearch/issues/28948 + assertNotNull(((ObjectMapper) mapping.getRoot().mappers.get("foo")).mappers.get("bar")); + assertNull(((ObjectMapper) mapping.getRoot().mappers.get("foo")).mappers.get("bar.")); + } + + public void testFieldStartingAndEndingWithDot() throws Exception { + XContentBuilder builder = mapping(b -> b.startObject("foo..bar.").field("type", "keyword").endObject()); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))) + ); + // TODO isn't this error misleading? + assertEquals("name cannot be empty string", iae.getMessage()); + } + + public void testDottedFieldWithTrailingWhitespace() throws Exception { + XContentBuilder builder = mapping(b -> b.startObject("top. .foo").field("type", "keyword").endObject()); + Mapping mapping = createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); + ObjectMapper top = (ObjectMapper) mapping.getRoot().mappers.get("top"); + // TODO this needs fixing? This field name is not allowed in documents when subobjects are enabled. + ObjectMapper mapper = (ObjectMapper) top.getMapper(" "); + assertNotNull(mapper.getMapper("foo")); + } + + public void testEmptyFieldName() throws Exception { + { + XContentBuilder builder = mapping(b -> b.startObject("").field("type", "keyword").endObject()); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))) + ); + assertEquals("name cannot be empty string", iae.getMessage()); + } + { + XContentBuilder builder = mappingNoSubobjects(b -> b.startObject("").field("type", "keyword").endObject()); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))) + ); + assertEquals("name cannot be empty string", iae.getMessage()); + } + } + + public void testBlankFieldName() throws Exception { + // TODO this needs fixing? This field name is never allowed in documents hence such a field can never be indexed? + { + XContentBuilder builder = mapping(b -> b.startObject(" ").field("type", "keyword").endObject()); + Mapping mapping = createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); + assertNotNull(mapping.getRoot().getMapper(" ")); + } + { + XContentBuilder builder = mappingNoSubobjects(b -> b.startObject(" ").field("type", "keyword").endObject()); + Mapping mapping = createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); + assertNotNull(mapping.getRoot().getMapper(" ")); + } + } + + public void testFieldNameDotsOnly() throws Exception { + String[] fieldNames = { ".", "..", "..." }; + for (String fieldName : fieldNames) { + XContentBuilder builder = mapping(b -> b.startObject(fieldName).field("type", "keyword").endObject()); + // TODO this should really throw a better error, relates to https://github.com/elastic/elasticsearch/issues/21862 + expectThrows( + ArrayIndexOutOfBoundsException.class, + () -> createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))) + ); + } + } + + public void testFieldNameDotsOnlySubobjectsFalse() throws Exception { + String[] fieldNames = { ".", "..", "..." }; + for (String fieldName : fieldNames) { + XContentBuilder builder = mappingNoSubobjects(b -> b.startObject(fieldName).field("type", "keyword").endObject()); + + createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); + } + } + + public void testDynamicFieldEdgeCaseNamesSubobjectsFalse() throws Exception { + // these combinations are not accepted by default, but they are when subobjects are disabled + String[] fieldNames = new String[] { ".foo", "foo.", "top..foo", "top.foo.", "top..foo.", "top. .foo" }; + MappingParser mappingParser = createMappingParser(Settings.EMPTY); + for (String fieldName : fieldNames) { + XContentBuilder builder = mappingNoSubobjects(b -> b.startObject(fieldName).field("type", "keyword").endObject()); + // TODO this is not accepted in documents, shall we revert https://github.com/elastic/elasticsearch/pull/90950 ? + assertNotNull(mappingParser.parse("_doc", new CompressedXContent(BytesReference.bytes(builder)))); + } + } + + public void testDynamicFieldEdgeCaseNamesRuntimeSection() throws Exception { + // TODO these combinations are not accepted by default, but they are in the runtime section, though they are not accepted when + // parsing documents with subobjects enabled + String[] fieldNames = new String[] { ".foo", "foo.", "top..foo", "top.foo.", "top..foo.", "top. .foo" }; + MappingParser mappingParser = createMappingParser(Settings.EMPTY); + for (String fieldName : fieldNames) { + XContentBuilder builder = runtimeMapping(b -> b.startObject(fieldName).field("type", "keyword").endObject()); + mappingParser.parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index c0b61a21a0938..326589ec59c42 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -424,17 +424,12 @@ public void testSubobjectsFalseWithInnerNested() { } public void testSubobjectsFalseRoot() throws Exception { - MapperService mapperService = createMapperService(topMapping(b -> { - b.field("subobjects", false); - b.startObject("properties"); - { - b.startObject("metrics.service.time"); - b.field("type", "long"); - b.endObject(); - b.startObject("metrics.service.time.max"); - b.field("type", "long"); - b.endObject(); - } + MapperService mapperService = createMapperService(mappingNoSubobjects(b -> { + b.startObject("metrics.service.time"); + b.field("type", "long"); + b.endObject(); + b.startObject("metrics.service.time.max"); + b.field("type", "long"); b.endObject(); })); assertNotNull(mapperService.fieldType("metrics.service.time")); @@ -447,18 +442,13 @@ public void testExplicitDefaultSubobjects() throws Exception { } public void testSubobjectsFalseRootWithInnerObject() { - MapperParsingException exception = expectThrows(MapperParsingException.class, () -> createMapperService(topMapping(b -> { - b.field("subobjects", false); - b.startObject("properties"); + MapperParsingException exception = expectThrows(MapperParsingException.class, () -> createMapperService(mappingNoSubobjects(b -> { + b.startObject("metrics.service.time"); { - b.startObject("metrics.service.time"); + b.startObject("properties"); { - b.startObject("properties"); - { - b.startObject("max"); - b.field("type", "long"); - b.endObject(); - } + b.startObject("max"); + b.field("type", "long"); b.endObject(); } b.endObject(); @@ -472,14 +462,9 @@ public void testSubobjectsFalseRootWithInnerObject() { } public void testSubobjectsFalseRootWithInnerNested() { - MapperParsingException exception = expectThrows(MapperParsingException.class, () -> createMapperService(topMapping(b -> { - b.field("subobjects", false); - b.startObject("properties"); - { - b.startObject("metrics.service"); - b.field("type", "nested"); - b.endObject(); - } + MapperParsingException exception = expectThrows(MapperParsingException.class, () -> createMapperService(mappingNoSubobjects(b -> { + b.startObject("metrics.service"); + b.field("type", "nested"); b.endObject(); }))); assertEquals( diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index c18026eb79b3c..7f8590445c721 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -314,6 +314,15 @@ protected final XContentBuilder topMapping(CheckedConsumer buildFields) throws IOException { + return topMapping(xContentBuilder -> { + xContentBuilder.field("subobjects", false); + xContentBuilder.startObject("properties"); + buildFields.accept(xContentBuilder); + xContentBuilder.endObject(); + }); + } + protected final XContentBuilder mapping(CheckedConsumer buildFields) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc").startObject("properties"); buildFields.accept(builder); From fee5c28694a72ec12f01210e975d9430dc2069a2 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 4 Nov 2022 12:24:29 +0100 Subject: [PATCH 05/17] Only add ReadTimeoutHandler to Netty4 http pipeline if not a noop (#91307) No need to add this for the (default) case of no http read timeout, it just adds needless overhead and makes profiling harder to read. --- .../http/netty4/Netty4HttpServerTransport.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index 1c6db3451f5a9..b2550d6ec42a6 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -307,21 +307,25 @@ protected HttpChannelHandler(final Netty4HttpServerTransport transport, final Ht protected void initChannel(Channel ch) throws Exception { Netty4HttpChannel nettyHttpChannel = new Netty4HttpChannel(ch); ch.attr(HTTP_CHANNEL_KEY).set(nettyHttpChannel); - ch.pipeline().addLast("chunked_writer", new Netty4WriteThrottlingHandler(transport.getThreadPool().getThreadContext())); - ch.pipeline().addLast("byte_buf_sizer", NettyByteBufSizer.INSTANCE); - ch.pipeline().addLast("read_timeout", new ReadTimeoutHandler(transport.readTimeoutMillis, TimeUnit.MILLISECONDS)); + ch.pipeline() + .addLast("chunked_writer", new Netty4WriteThrottlingHandler(transport.getThreadPool().getThreadContext())) + .addLast("byte_buf_sizer", NettyByteBufSizer.INSTANCE); + if (transport.readTimeoutMillis > 0) { + ch.pipeline().addLast("read_timeout", new ReadTimeoutHandler(transport.readTimeoutMillis, TimeUnit.MILLISECONDS)); + } final HttpRequestDecoder decoder = new HttpRequestDecoder( handlingSettings.maxInitialLineLength(), handlingSettings.maxHeaderSize(), handlingSettings.maxChunkSize() ); decoder.setCumulator(ByteToMessageDecoder.COMPOSITE_CUMULATOR); - ch.pipeline().addLast("decoder", decoder); - ch.pipeline().addLast("decoder_compress", new HttpContentDecompressor()); - ch.pipeline().addLast("encoder", new HttpResponseEncoder()); final HttpObjectAggregator aggregator = new HttpObjectAggregator(handlingSettings.maxContentLength()); aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); - ch.pipeline().addLast("aggregator", aggregator); + ch.pipeline() + .addLast("decoder", decoder) + .addLast("decoder_compress", new HttpContentDecompressor()) + .addLast("encoder", new HttpResponseEncoder()) + .addLast("aggregator", aggregator); if (handlingSettings.compression()) { ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(handlingSettings.compressionLevel())); } From 4e67df8b0512ceb63f529539ed78d27c707a737b Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 4 Nov 2022 14:22:30 +0200 Subject: [PATCH 06/17] [ML] Low priority trained model deployments (#91234) This adds a new parameter to the start trained model deployment API, namely `priority`. The available settings are `normal` and `low`. For normal priority deployments the allocations get distributed so that node processors are never oversubscribed. Low priority deployments allow users to test model functionality even if there are no node processors available. They are limited to 1 allocation with a single thread. In addition, the process is executed in low priority which limits the amount of CPU that can be used when the CPU is under pressure. The intention of this is to limit the impact of low priority deployments on normal priority deployments. When we rebalance model assignments we now: 1. compute a plan just for normal priority deployments 2. fix the resources used by normal deployments 3. compute a plan just for low priority deployments 4. merge the two plans Closes #91024 --- docs/changelog/91234.yaml | 6 + .../apis/get-trained-models-stats.asciidoc | 4 + .../start-trained-model-deployment.asciidoc | 23 +- .../ml.start_trained_model_deployment.json | 6 + .../StartTrainedModelDeploymentAction.java | 77 ++- .../inference/assignment/AssignmentStats.java | 24 +- .../ml/inference/assignment/Priority.java | 24 + .../assignment/TrainedModelAssignment.java | 3 +- ...TrainedModelsStatsActionResponseTests.java | 70 ++- ...artTrainedModelDeploymentRequestTests.java | 30 ++ ...TrainedModelDeploymentTaskParamsTests.java | 4 +- .../assignment/AssignmentStatsTests.java | 12 +- .../inference/assignment/PriorityTests.java | 22 + .../TrainedModelAssignmentTests.java | 3 +- .../ml/qa/ml-with-security/build.gradle | 2 + .../TransportGetDeploymentStatsAction.java | 9 +- ...portStartTrainedModelDeploymentAction.java | 3 +- .../MlProcessorAutoscalingDecider.java | 14 +- .../TrainedModelAssignmentNodeService.java | 3 +- .../TrainedModelAssignmentRebalancer.java | 198 ++++++- .../assignment/planning/AssignmentPlan.java | 34 +- .../planning/AssignmentPlanner.java | 6 +- .../planning/ZoneAwareAssignmentPlanner.java | 21 +- .../TrainedModelDeploymentTask.java | 3 +- .../ChunkedTrainedModelRestorer.java | 1 + .../process/NativePyTorchProcessFactory.java | 8 +- .../pytorch/process/PyTorchBuilder.java | 26 +- ...RestStartTrainedModelDeploymentAction.java | 6 + ...chineLearningInfoTransportActionTests.java | 6 +- ...ransportGetDeploymentStatsActionTests.java | 9 +- .../MlMemoryAutoscalingDeciderTests.java | 13 +- .../MlProcessorAutoscalingDeciderTests.java | 238 ++++++++- ...nedModelAssignmentClusterServiceTests.java | 4 +- .../TrainedModelAssignmentMetadataTests.java | 4 +- ...rainedModelAssignmentNodeServiceTests.java | 4 +- ...TrainedModelAssignmentRebalancerTests.java | 482 +++++++++++++++++- .../planning/AllocationReducerTests.java | 4 +- .../ZoneAwareAssignmentPlannerTests.java | 3 + .../TrainedModelDeploymentTaskTests.java | 8 +- .../pytorch/process/PyTorchBuilderTests.java | 36 +- .../xpack/ml/job/NodeLoadDetectorTests.java | 4 +- .../test/ml/3rd_party_deployment.yml | 33 ++ 42 files changed, 1339 insertions(+), 151 deletions(-) create mode 100644 docs/changelog/91234.yaml create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/Priority.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/PriorityTests.java diff --git a/docs/changelog/91234.yaml b/docs/changelog/91234.yaml new file mode 100644 index 0000000000000..1e25efe5aa301 --- /dev/null +++ b/docs/changelog/91234.yaml @@ -0,0 +1,6 @@ +pr: 91234 +summary: Low priority trained model deployments +area: Machine Learning +type: enhancement +issues: + - 91024 diff --git a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc index b9751b5a9ce96..5dd7420ee17f0 100644 --- a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc +++ b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc @@ -271,6 +271,10 @@ The peak number of requests processed in a 1 minute period for all nodes in the deployment. This is calculated as the sum of each node's `peak_throughput_per_minute` value. +`priority`::: +(string) +The deployment priority. + `rejected_execution_count`::: (integer) The sum of `rejected_execution_count` for all nodes in the deployment. diff --git a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc index baf2e086c3421..f5320fe8667f5 100644 --- a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc @@ -66,6 +66,26 @@ The total number of allocations this model is assigned across {ml} nodes. Increasing this value generally increases the throughput. Defaults to 1. +`priority`:: +(Optional, string) +The priority of the deployment. The default value is `normal`. + +There are two priority settings: ++ +-- +* `normal`: Use this for deployments in production. +The deployment allocations are distributed so that node processors are not oversubscribed. +* `low`: Use this for testing model functionality. +The intention is that these deployments are not sent a high volume of input. +The deployment is required to have a single allocation with just one thread. +Low priority deployments may be assigned on nodes that already utilize all their processors +but will be given a lower CPU priority than normal deployments. Low priority deployments may be unassigned in order +to satisfy more allocations of normal priority deployments. +-- + +WARNING: Heavy usage of low priority deployments may impact performance of normal +priority deployments. + `queue_capacity`:: (Optional, integer) Controls how many inference requests are allowed in the queue at a time. @@ -116,7 +136,8 @@ The API returns the following results: "model_bytes": 265632637, "threads_per_allocation" : 1, "number_of_allocations" : 1, - "queue_capacity" : 1024 + "queue_capacity" : 1024, + "priority": "normal" }, "routing_table": { "uckeG3R8TLe2MMNBQ6AGrw": { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json index 9478c735cb0d4..db34187598984 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json @@ -45,6 +45,12 @@ "required": false, "default": 1 }, + "priority": { + "type": "string", + "description": "The deployment priority.", + "required": false, + "default": "normal" + }, "queue_capacity":{ "type":"int", "description": "Controls how many inference requests are allowed in the queue at a time.", diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java index 557c45a7f0ae3..ddb01225e6663 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.MlTaskParams; @@ -83,6 +84,7 @@ public static class Request extends MasterNodeRequest implements ToXCon public static final ParseField NUMBER_OF_ALLOCATIONS = new ParseField("number_of_allocations", "model_threads"); public static final ParseField QUEUE_CAPACITY = TaskParams.QUEUE_CAPACITY; public static final ParseField CACHE_SIZE = TaskParams.CACHE_SIZE; + public static final ParseField PRIORITY = TaskParams.PRIORITY; public static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); @@ -99,6 +101,7 @@ public static class Request extends MasterNodeRequest implements ToXCon CACHE_SIZE, ObjectParser.ValueType.VALUE ); + PARSER.declareString(Request::setPriority, PRIORITY); } public static Request parseRequest(String modelId, XContentParser parser) { @@ -120,6 +123,7 @@ public static Request parseRequest(String modelId, XContentParser parser) { private int numberOfAllocations = 1; private int threadsPerAllocation = 1; private int queueCapacity = 1024; + private Priority priority = Priority.NORMAL; private Request() {} @@ -138,6 +142,11 @@ public Request(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_8_4_0)) { this.cacheSize = in.readOptionalWriteable(ByteSizeValue::readFrom); } + if (in.getVersion().onOrAfter(Version.V_8_6_0)) { + this.priority = in.readEnum(Priority.class); + } else { + this.priority = Priority.NORMAL; + } } public final void setModelId(String modelId) { @@ -197,6 +206,14 @@ public void setCacheSize(ByteSizeValue cacheSize) { this.cacheSize = cacheSize; } + public Priority getPriority() { + return priority; + } + + public void setPriority(String priority) { + this.priority = Priority.fromString(priority); + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -209,6 +226,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_8_4_0)) { out.writeOptionalWriteable(cacheSize); } + if (out.getVersion().onOrAfter(Version.V_8_6_0)) { + out.writeEnum(priority); + } } @Override @@ -223,6 +243,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (cacheSize != null) { builder.field(CACHE_SIZE.getPreferredName(), cacheSize); } + builder.field(PRIORITY.getPreferredName(), priority); builder.endObject(); return builder; } @@ -258,6 +279,14 @@ public ActionRequestValidationException validate() { if (timeout.nanos() < 1L) { validationException.addValidationError("[" + TIMEOUT + "] must be positive"); } + if (priority == Priority.LOW) { + if (numberOfAllocations > 1) { + validationException.addValidationError("[" + NUMBER_OF_ALLOCATIONS + "] must be 1 when [" + PRIORITY + "] is low"); + } + if (threadsPerAllocation > 1) { + validationException.addValidationError("[" + THREADS_PER_ALLOCATION + "] must be 1 when [" + PRIORITY + "] is low"); + } + } return validationException.validationErrors().isEmpty() ? null : validationException; } @@ -267,7 +296,16 @@ private static boolean isPowerOf2(int value) { @Override public int hashCode() { - return Objects.hash(modelId, timeout, waitForState, numberOfAllocations, threadsPerAllocation, queueCapacity, cacheSize); + return Objects.hash( + modelId, + timeout, + waitForState, + numberOfAllocations, + threadsPerAllocation, + queueCapacity, + cacheSize, + priority + ); } @Override @@ -285,7 +323,8 @@ public boolean equals(Object obj) { && Objects.equals(cacheSize, other.cacheSize) && numberOfAllocations == other.numberOfAllocations && threadsPerAllocation == other.threadsPerAllocation - && queueCapacity == other.queueCapacity; + && queueCapacity == other.queueCapacity + && priority == other.priority; } @Override @@ -313,6 +352,7 @@ public static boolean mayAssignToNode(DiscoveryNode node) { public static final ParseField LEGACY_INFERENCE_THREADS = new ParseField("inference_threads"); public static final ParseField QUEUE_CAPACITY = new ParseField("queue_capacity"); public static final ParseField CACHE_SIZE = new ParseField("cache_size"); + public static final ParseField PRIORITY = new ParseField("priority"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "trained_model_deployment_params", @@ -325,7 +365,8 @@ public static boolean mayAssignToNode(DiscoveryNode node) { (int) a[4], (ByteSizeValue) a[5], (Integer) a[6], - (Integer) a[7] + (Integer) a[7], + a[8] == null ? null : Priority.fromString((String) a[8]) ) ); @@ -343,6 +384,7 @@ public static boolean mayAssignToNode(DiscoveryNode node) { ); PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), LEGACY_MODEL_THREADS); PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), LEGACY_INFERENCE_THREADS); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PRIORITY); } public static TaskParams fromXContent(XContentParser parser) { @@ -357,6 +399,7 @@ public static TaskParams fromXContent(XContentParser parser) { // How many threads are used when forwarding the request to the model. Used to increase throughput. private final int numberOfAllocations; private final int queueCapacity; + private final Priority priority; private TaskParams( String modelId, @@ -366,7 +409,8 @@ private TaskParams( int queueCapacity, ByteSizeValue cacheSizeValue, Integer legacyModelThreads, - Integer legacyInferenceThreads + Integer legacyInferenceThreads, + Priority priority ) { this( modelId, @@ -374,7 +418,8 @@ private TaskParams( numberOfAllocations == null ? legacyModelThreads : numberOfAllocations, threadsPerAllocation == null ? legacyInferenceThreads : threadsPerAllocation, queueCapacity, - cacheSizeValue + cacheSizeValue, + priority == null ? Priority.NORMAL : priority ); } @@ -384,7 +429,8 @@ public TaskParams( int numberOfAllocations, int threadsPerAllocation, int queueCapacity, - @Nullable ByteSizeValue cacheSize + @Nullable ByteSizeValue cacheSize, + Priority priority ) { this.modelId = Objects.requireNonNull(modelId); this.modelBytes = modelBytes; @@ -392,6 +438,7 @@ public TaskParams( this.numberOfAllocations = numberOfAllocations; this.queueCapacity = queueCapacity; this.cacheSize = cacheSize; + this.priority = Objects.requireNonNull(priority); } public TaskParams(StreamInput in) throws IOException { @@ -405,6 +452,11 @@ public TaskParams(StreamInput in) throws IOException { } else { this.cacheSize = null; } + if (in.getVersion().onOrAfter(Version.V_8_6_0)) { + this.priority = in.readEnum(Priority.class); + } else { + this.priority = Priority.NORMAL; + } } public String getModelId() { @@ -434,6 +486,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_8_4_0)) { out.writeOptionalWriteable(cacheSize); } + if (out.getVersion().onOrAfter(Version.V_8_6_0)) { + out.writeEnum(priority); + } } @Override @@ -447,13 +502,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (cacheSize != null) { builder.field(CACHE_SIZE.getPreferredName(), cacheSize.getStringRep()); } + builder.field(PRIORITY.getPreferredName(), priority); builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(modelId, modelBytes, threadsPerAllocation, numberOfAllocations, queueCapacity, cacheSize); + return Objects.hash(modelId, modelBytes, threadsPerAllocation, numberOfAllocations, queueCapacity, cacheSize, priority); } @Override @@ -467,7 +523,8 @@ public boolean equals(Object o) { && threadsPerAllocation == other.threadsPerAllocation && numberOfAllocations == other.numberOfAllocations && Objects.equals(cacheSize, other.cacheSize) - && queueCapacity == other.queueCapacity; + && queueCapacity == other.queueCapacity + && priority == other.priority; } @Override @@ -499,6 +556,10 @@ public long getCacheSizeBytes() { return Optional.ofNullable(cacheSize).map(ByteSizeValue::getBytes).orElse(modelBytes); } + public Priority getPriority() { + return priority; + } + @Override public String toString() { return Strings.toString(this); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java index d054737799b51..05f62f216c10c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java @@ -425,6 +425,7 @@ public int hashCode() { private final Integer queueCapacity; @Nullable private final ByteSizeValue cacheSize; + private final Priority priority; private final Instant startTime; private final List nodeStats; @@ -435,7 +436,8 @@ public AssignmentStats( @Nullable Integer queueCapacity, @Nullable ByteSizeValue cacheSize, Instant startTime, - List nodeStats + List nodeStats, + Priority priority ) { this.modelId = modelId; this.threadsPerAllocation = threadsPerAllocation; @@ -446,6 +448,7 @@ public AssignmentStats( this.cacheSize = cacheSize; this.state = null; this.reason = null; + this.priority = Objects.requireNonNull(priority); } public AssignmentStats(StreamInput in) throws IOException { @@ -463,6 +466,11 @@ public AssignmentStats(StreamInput in) throws IOException { } else { cacheSize = null; } + if (in.getVersion().onOrAfter(Version.V_8_6_0)) { + priority = in.readEnum(Priority.class); + } else { + priority = Priority.NORMAL; + } } public String getModelId() { @@ -520,6 +528,10 @@ public AssignmentStats setReason(String reason) { return this; } + public Priority getPriority() { + return priority; + } + /** * @return The overall inference stats for the model assignment */ @@ -565,6 +577,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (cacheSize != null) { builder.field("cache_size", cacheSize); } + builder.field("priority", priority); builder.timeField("start_time", "start_time_string", startTime.toEpochMilli()); int totalErrorCount = nodeStats.stream().mapToInt(NodeStats::getErrorCount).sum(); @@ -617,6 +630,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_8_4_0)) { out.writeOptionalWriteable(cacheSize); } + if (out.getVersion().onOrAfter(Version.V_8_6_0)) { + out.writeEnum(priority); + } } @Override @@ -633,7 +649,8 @@ public boolean equals(Object o) { && Objects.equals(reason, that.reason) && Objects.equals(allocationStatus, that.allocationStatus) && Objects.equals(cacheSize, that.cacheSize) - && Objects.equals(nodeStats, that.nodeStats); + && Objects.equals(nodeStats, that.nodeStats) + && priority == that.priority; } @Override @@ -648,7 +665,8 @@ public int hashCode() { state, reason, allocationStatus, - cacheSize + cacheSize, + priority ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/Priority.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/Priority.java new file mode 100644 index 0000000000000..feeca92462c31 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/Priority.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.assignment; + +import java.util.Locale; + +public enum Priority { + LOW, + NORMAL; + + public static Priority fromString(String value) { + return valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java index 084efa7a7a891..901416a71b513 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java @@ -437,7 +437,8 @@ public Builder setNumberOfAllocations(int numberOfAllocations) { numberOfAllocations, taskParams.getThreadsPerAllocation(), taskParams.getQueueCapacity(), - taskParams.getCacheSize().orElse(null) + taskParams.getCacheSize().orElse(null), + taskParams.getPriority() ); return this; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java index 04813065d1025..97d567939d590 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction.Response; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStatsTests; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceStatsTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModelSizeStatsTests; @@ -141,7 +142,8 @@ protected Response mutateInstanceForVersion(Response instance, Version version) null ) ) - .toList() + .toList(), + Priority.NORMAL ) ) ) @@ -197,7 +199,8 @@ protected Response mutateInstanceForVersion(Response instance, Version version) null ) ) - .toList() + .toList(), + Priority.NORMAL ) ) ) @@ -253,7 +256,8 @@ protected Response mutateInstanceForVersion(Response instance, Version version) null ) ) - .toList() + .toList(), + Priority.NORMAL ) ) ) @@ -309,7 +313,65 @@ protected Response mutateInstanceForVersion(Response instance, Version version) nodeStats.getCacheHitCountLastPeriod().orElse(null) ) ) - .toList() + .toList(), + Priority.NORMAL + ) + ) + ) + .toList(), + instance.getResources().count(), + RESULTS_FIELD + ) + ); + } else if (version.before(Version.V_8_6_0)) { + return new Response( + new QueryPage<>( + instance.getResources() + .results() + .stream() + .map( + stats -> new Response.TrainedModelStats( + stats.getModelId(), + stats.getModelSizeStats(), + stats.getIngestStats(), + stats.getPipelineCount(), + stats.getInferenceStats(), + stats.getDeploymentStats() == null + ? null + : new AssignmentStats( + stats.getDeploymentStats().getModelId(), + stats.getDeploymentStats().getThreadsPerAllocation(), + stats.getDeploymentStats().getNumberOfAllocations(), + stats.getDeploymentStats().getQueueCapacity(), + stats.getDeploymentStats().getCacheSize(), + stats.getDeploymentStats().getStartTime(), + stats.getDeploymentStats() + .getNodeStats() + .stream() + .map( + nodeStats -> new AssignmentStats.NodeStats( + nodeStats.getNode(), + nodeStats.getInferenceCount().orElse(null), + nodeStats.getAvgInferenceTime().orElse(null), + nodeStats.getAvgInferenceTimeExcludingCacheHit().orElse(null), + nodeStats.getLastAccess(), + nodeStats.getPendingCount(), + nodeStats.getErrorCount(), + nodeStats.getCacheHitCount().orElse(null), + nodeStats.getRejectedExecutionCount(), + nodeStats.getTimeoutCount(), + nodeStats.getRoutingState(), + nodeStats.getStartTime(), + nodeStats.getThreadsPerAllocation(), + nodeStats.getNumberOfAllocations(), + nodeStats.getPeakThroughput(), + nodeStats.getThroughputLastPeriod(), + nodeStats.getAvgInferenceTimeLastPeriod(), + nodeStats.getCacheHitCountLastPeriod().orElse(null) + ) + ) + .toList(), + Priority.NORMAL ) ) ) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java index 609b6f9c51d00..ea59544a1c9ee 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.Request; import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import java.io.IOException; import java.util.List; @@ -61,6 +62,12 @@ public static Request createRandom() { if (randomBoolean()) { request.setQueueCapacity(randomIntBetween(1, 1000000)); } + if (randomBoolean()) { + request.setPriority(randomFrom(Priority.values()).toString()); + if (request.getNumberOfAllocations() > 1 || request.getThreadsPerAllocation() > 1) { + request.setPriority(Priority.NORMAL.toString()); + } + } return request; } @@ -102,6 +109,7 @@ public void testValidate_GivenThreadsPerAllocationIsNotPowerOf2() { public void testValidate_GivenThreadsPerAllocationIsValid() { for (int n : List.of(1, 2, 4, 8, 16, 32)) { Request request = createRandom(); + request.setPriority(Priority.NORMAL.toString()); request.setThreadsPerAllocation(n); ActionRequestValidationException e = request.validate(); @@ -189,6 +197,28 @@ public void testValidate_GivenTimeoutIsZero() { assertThat(e.getMessage(), containsString("[timeout] must be positive")); } + public void testValidate_GivenLowPriorityAndMultipleThreadsPerAllocation() { + Request request = createRandom(); + request.setPriority(Priority.LOW.toString()); + request.setThreadsPerAllocation(randomFrom(2, 4, 8, 16, 32)); + + ActionRequestValidationException e = request.validate(); + + assertThat(e, is(not(nullValue()))); + assertThat(e.getMessage(), containsString("[threads_per_allocation] must be 1 when [priority] is low")); + } + + public void testValidate_GivenLowPriorityAndMultipleAllocations() { + Request request = createRandom(); + request.setPriority(Priority.LOW.toString()); + request.setNumberOfAllocations(randomIntBetween(2, 32)); + + ActionRequestValidationException e = request.validate(); + + assertThat(e, is(not(nullValue()))); + assertThat(e.getMessage(), containsString("[number_of_allocations] must be 1 when [priority] is low")); + } + public void testDefaults() { Request request = new Request(randomAlphaOfLength(10)); assertThat(request.getTimeout(), equalTo(TimeValue.timeValueSeconds(20))); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java index 8d9549a18d438..3b8ad17daa58e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.TaskParams; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import java.io.IOException; @@ -39,7 +40,8 @@ public static StartTrainedModelDeploymentAction.TaskParams createRandom() { randomIntBetween(1, 8), randomIntBetween(1, 8), randomIntBetween(1, 10000), - randomBoolean() ? null : ByteSizeValue.ofBytes(randomNonNegativeLong()) + randomBoolean() ? null : ByteSizeValue.ofBytes(randomNonNegativeLong()), + randomFrom(Priority.values()) ); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStatsTests.java index f723d8c1c8eab..0b31dff8c2077 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStatsTests.java @@ -50,7 +50,8 @@ public static AssignmentStats randomDeploymentStats() { randomBoolean() ? null : randomIntBetween(1, 10000), randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, 10000000)), Instant.now(), - nodeStatsList + nodeStatsList, + randomFrom(Priority.values()) ); } @@ -144,7 +145,8 @@ public void testGetOverallInferenceStats() { randomFrom(RoutingState.values()), randomBoolean() ? null : "a good reason" ) - ) + ), + randomFrom(Priority.values()) ); InferenceStats stats = existingStats.getOverallInferenceStats(); assertThat(stats.getModelId(), equalTo(modelId)); @@ -162,7 +164,8 @@ public void testGetOverallInferenceStatsWithNoNodes() { randomBoolean() ? null : randomIntBetween(1, 10000), randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, 1000000)), Instant.now(), - List.of() + List.of(), + randomFrom(Priority.values()) ); InferenceStats stats = existingStats.getOverallInferenceStats(); assertThat(stats.getModelId(), equalTo(modelId)); @@ -191,7 +194,8 @@ public void testGetOverallInferenceStatsWithOnlyStoppedNodes() { randomFrom(RoutingState.values()), randomBoolean() ? null : "a good reason" ) - ) + ), + randomFrom(Priority.values()) ); InferenceStats stats = existingStats.getOverallInferenceStats(); assertThat(stats.getModelId(), equalTo(modelId)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/PriorityTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/PriorityTests.java new file mode 100644 index 0000000000000..c3cca606d69f5 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/PriorityTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.assignment; + +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class PriorityTests extends ESTestCase { + + public void testToAndFromString() { + for (Priority priority : Priority.values()) { + String value = priority.toString(); + assertThat(Priority.fromString(value), equalTo(priority)); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java index 5229b08bb42b0..bb6d1904a99ab 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java @@ -289,7 +289,8 @@ private static StartTrainedModelDeploymentAction.TaskParams randomTaskParams(int numberOfAllocations, randomIntBetween(1, 8), randomIntBetween(1, 10000), - randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(0, modelSize + 1)) + randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(0, modelSize + 1)), + randomFrom(Priority.values()) ); } diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index d6a0442766be9..478736168d330 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -28,6 +28,8 @@ tasks.named("yamlRestTest").configure { // Remove tests that are expected to throw an exception, because we cannot then // know whether to expect an authorization exception or a validation exception 'ml/3rd_party_deployment/Test start deployment fails with missing model definition', + 'ml/3rd_party_deployment/Test start deployment with low priority and multiple allocations', + 'ml/3rd_party_deployment/Test start deployment with low priority and multiple threads per allocation', 'ml/calendar_crud/Test get calendar given missing', 'ml/calendar_crud/Test cannot create calendar with name _all', 'ml/calendar_crud/Test PageParams with ID is invalid', diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java index 4f8fbf544d93f..2e1e031d4fff9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java @@ -240,7 +240,8 @@ static GetDeploymentStatsAction.Response addFailedRoutes( stat.getQueueCapacity(), stat.getCacheSize(), stat.getStartTime(), - updatedNodeStats + updatedNodeStats, + stat.getPriority() ) ); } else { @@ -277,7 +278,8 @@ static GetDeploymentStatsAction.Response addFailedRoutes( assignment.getTaskParams().getQueueCapacity(), assignment.getTaskParams().getCacheSize().orElse(null), assignment.getStartTime(), - nodeStats + nodeStats, + assignment.getTaskParams().getPriority() ) ); } @@ -344,7 +346,8 @@ protected void taskOperation( task.getParams().getQueueCapacity(), task.getParams().getCacheSize().orElse(null), TrainedModelAssignmentMetadata.fromState(clusterService.state()).getModelAssignment(task.getModelId()).getStartTime(), - nodeStats + nodeStats, + task.getParams().getPriority() ) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java index b748da3fa1d6c..2ec93a0d6bf95 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java @@ -212,7 +212,8 @@ protected void masterOperation( request.getNumberOfAllocations(), request.getThreadsPerAllocation(), request.getQueueCapacity(), - Optional.ofNullable(request.getCacheSize()).orElse(ByteSizeValue.ofBytes(modelBytes)) + Optional.ofNullable(request.getCacheSize()).orElse(ByteSizeValue.ofBytes(modelBytes)), + request.getPriority() ); PersistentTasksCustomMetadata persistentTasks = clusterService.state() .getMetadata() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlProcessorAutoscalingDecider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlProcessorAutoscalingDecider.java index 772fd2a847d30..0b6e8aa950624 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlProcessorAutoscalingDecider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlProcessorAutoscalingDecider.java @@ -15,6 +15,7 @@ import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderContext; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentMetadata; import org.elasticsearch.xpack.ml.utils.MlProcessors; @@ -106,18 +107,29 @@ private boolean hasUnsatisfiedDeployments(TrainedModelAssignmentMetadata trained return trainedModelAssignmentMetadata.modelAssignments() .values() .stream() + .filter(deployment -> deployment.getTaskParams().getPriority() == Priority.NORMAL) .anyMatch(deployment -> deployment.isSatisfied(mlNodeIds) == false); } private MlProcessorAutoscalingCapacity.Builder computeRequiredCapacity(TrainedModelAssignmentMetadata trainedModelAssignmentMetadata) { int maxThreadsPerAllocation = 0; - int processorCount = 0; + double processorCount = 0; + boolean hasLowPriorityDeployments = false; for (TrainedModelAssignment assignment : trainedModelAssignmentMetadata.modelAssignments().values()) { + if (assignment.getTaskParams().getPriority() == Priority.LOW) { + hasLowPriorityDeployments = true; + continue; + } int threadsPerAllocation = assignment.getTaskParams().getThreadsPerAllocation(); maxThreadsPerAllocation = Math.max(maxThreadsPerAllocation, threadsPerAllocation); processorCount += assignment.getTaskParams().getNumberOfAllocations() * threadsPerAllocation; } + if (hasLowPriorityDeployments) { + // If there are low priority deployments let us ensure there will at least be one node required. + processorCount = Math.max(0.1, processorCount); + } + return MlProcessorAutoscalingCapacity.builder( maxThreadsPerAllocation > 0 ? Processors.of(Double.valueOf(maxThreadsPerAllocation)) : Processors.ZERO, processorCount > 0 ? Processors.of(Double.valueOf(processorCount)) : Processors.ZERO diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java index 0ef20a9f99e5a..8a1f818ed22f6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java @@ -363,7 +363,8 @@ public void clusterChanged(ClusterChangedEvent event) { routingInfo.getCurrentAllocations(), trainedModelAssignment.getTaskParams().getThreadsPerAllocation(), trainedModelAssignment.getTaskParams().getQueueCapacity(), - trainedModelAssignment.getTaskParams().getCacheSize().orElse(null) + trainedModelAssignment.getTaskParams().getCacheSize().orElse(null), + trainedModelAssignment.getTaskParams().getPriority() ) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentRebalancer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentRebalancer.java index 8da81ae5396a7..ba156fefdfa75 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentRebalancer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentRebalancer.java @@ -15,23 +15,28 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.inference.assignment.planning.AssignmentPlan; +import org.elasticsearch.xpack.ml.inference.assignment.planning.AssignmentPlanner; import org.elasticsearch.xpack.ml.inference.assignment.planning.ZoneAwareAssignmentPlanner; import org.elasticsearch.xpack.ml.job.NodeLoad; import org.elasticsearch.xpack.ml.utils.MlProcessors; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; @@ -40,6 +45,13 @@ class TrainedModelAssignmentRebalancer { private static final Logger logger = LogManager.getLogger(TrainedModelAssignmentRebalancer.class); + /** + * We set the max number of low priority models per node to 100, + * a value that effectively removes the processor constraint and + * transforms the problem to memory bin packing. + */ + private static final int MAX_LOW_PRIORITY_MODELS_PER_NODE = 100; + private final TrainedModelAssignmentMetadata currentMetadata; private final Map nodeLoads; private final Map, Collection> mlNodesByZone; @@ -83,36 +95,89 @@ private boolean areAllModelsSatisfiedAndNoOutdatedRoutingEntries() { AssignmentPlan computeAssignmentPlan() { final Map, List> nodesByZone = createNodesByZoneMap(); - - final List planModels = new ArrayList<>( - currentMetadata.modelAssignments().size() + (modelToAdd.isPresent() ? 1 : 0) - ); final Set assignableNodeIds = nodesByZone.values() .stream() .flatMap(List::stream) .map(AssignmentPlan.Node::id) .collect(Collectors.toSet()); - currentMetadata.modelAssignments().values().stream().map(assignment -> { - Map currentAssignments = assignment.getNodeRoutingTable() - .entrySet() - .stream() - // Filter out nodes that are no longer assignable - .filter(e -> assignableNodeIds.contains(e.getKey())) - // Filter out allocation without current and target allocations as they are from before using the rebalancer - .filter(e -> e.getValue().getCurrentAllocations() > 0 && e.getValue().getTargetAllocations() > 0) - .filter(e -> e.getValue().getState().isAnyOf(RoutingState.STARTING, RoutingState.STARTED, RoutingState.FAILED)) - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getTargetAllocations())); - return new AssignmentPlan.Model( - assignment.getModelId(), - assignment.getTaskParams().estimateMemoryUsageBytes(), - assignment.getTaskParams().getNumberOfAllocations(), - assignment.getTaskParams().getThreadsPerAllocation(), - currentAssignments, - assignment.getMaxAssignedAllocations() - ); - }).forEach(planModels::add); - modelToAdd.ifPresent( - taskParams -> planModels.add( + + AssignmentPlan planForNormalPriorityModels = computePlanForNormalPriorityModels(nodesByZone, assignableNodeIds); + AssignmentPlan planForLowPriorityModels = computePlanForLowPriorityModels(assignableNodeIds, planForNormalPriorityModels); + return mergePlans(nodesByZone, planForNormalPriorityModels, planForLowPriorityModels); + } + + private AssignmentPlan mergePlans( + Map, List> nodesByZone, + AssignmentPlan planForNormalPriorityModels, + AssignmentPlan planForLowPriorityModels + ) { + final List allNodes = new ArrayList<>(); + nodesByZone.values().forEach(allNodes::addAll); + + final List allModels = new ArrayList<>(); + allModels.addAll(planForNormalPriorityModels.models()); + allModels.addAll(planForLowPriorityModels.models()); + + final Map originalNodeById = allNodes.stream() + .collect(Collectors.toMap(AssignmentPlan.Node::id, Function.identity())); + AssignmentPlan.Builder finalPlanBuilder = AssignmentPlan.builder(allNodes, allModels); + copyAssignments(planForNormalPriorityModels, finalPlanBuilder, originalNodeById); + copyAssignments(planForLowPriorityModels, finalPlanBuilder, originalNodeById); + return finalPlanBuilder.build(); + } + + private static void copyAssignments( + AssignmentPlan source, + AssignmentPlan.Builder dest, + Map originalNodeById + ) { + for (AssignmentPlan.Model m : source.models()) { + Map nodeAssignments = source.assignments(m).orElse(Map.of()); + for (Map.Entry assignment : nodeAssignments.entrySet()) { + AssignmentPlan.Node originalNode = originalNodeById.get(assignment.getKey().id()); + dest.assignModelToNode(m, originalNode, assignment.getValue()); + if (m.currentAllocationsByNodeId().containsKey(originalNode.id())) { + // As the node has all its available memory we need to manually account memory of models with + // current allocations. + dest.accountMemory(m, originalNode); + } + } + } + } + + private AssignmentPlan computePlanForNormalPriorityModels( + Map, List> nodesByZone, + Set assignableNodeIds + ) { + final List planModels = new ArrayList<>(); + + currentMetadata.modelAssignments() + .values() + .stream() + .filter(assignment -> assignment.getTaskParams().getPriority() != Priority.LOW) + .map(assignment -> { + Map currentAssignments = assignment.getNodeRoutingTable() + .entrySet() + .stream() + // Filter out nodes that are no longer assignable + .filter(e -> assignableNodeIds.contains(e.getKey())) + // Filter out allocation without current and target allocations as they are from before using the rebalancer + .filter(e -> e.getValue().getCurrentAllocations() > 0 && e.getValue().getTargetAllocations() > 0) + .filter(e -> e.getValue().getState().isAnyOf(RoutingState.STARTING, RoutingState.STARTED, RoutingState.FAILED)) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getTargetAllocations())); + return new AssignmentPlan.Model( + assignment.getModelId(), + assignment.getTaskParams().estimateMemoryUsageBytes(), + assignment.getTaskParams().getNumberOfAllocations(), + assignment.getTaskParams().getThreadsPerAllocation(), + currentAssignments, + assignment.getMaxAssignedAllocations() + ); + }) + .forEach(planModels::add); + if (modelToAdd.isPresent() && modelToAdd.get().getPriority() != Priority.LOW) { + StartTrainedModelDeploymentAction.TaskParams taskParams = modelToAdd.get(); + planModels.add( new AssignmentPlan.Model( taskParams.getModelId(), taskParams.estimateMemoryUsageBytes(), @@ -121,9 +186,90 @@ AssignmentPlan computeAssignmentPlan() { Map.of(), 0 ) + ); + } + return new ZoneAwareAssignmentPlanner(nodesByZone, planModels).computePlan(); + } + + private AssignmentPlan computePlanForLowPriorityModels(Set assignableNodeIds, AssignmentPlan planExcludingLowPriorityModels) { + List planNodes = mlNodesByZone.values() + .stream() + .flatMap(Collection::stream) + .map( + discoveryNode -> new AssignmentPlan.Node( + discoveryNode.getId(), + planExcludingLowPriorityModels.getRemainingNodeMemory(discoveryNode.getId()), + MAX_LOW_PRIORITY_MODELS_PER_NODE + ) ) + .toList(); + + final Map remainingNodeMemory = new HashMap<>(); + planNodes.forEach(n -> remainingNodeMemory.put(n.id(), n.availableMemoryBytes())); + + final List planModels = new ArrayList<>(); + currentMetadata.modelAssignments() + .values() + .stream() + .filter(assignment -> assignment.getTaskParams().getPriority() == Priority.LOW) + .sorted(Comparator.comparingLong(assignment -> assignment.getTaskParams().estimateMemoryUsageBytes())) + .map( + assignment -> new AssignmentPlan.Model( + assignment.getModelId(), + assignment.getTaskParams().estimateMemoryUsageBytes(), + assignment.getTaskParams().getNumberOfAllocations(), + assignment.getTaskParams().getThreadsPerAllocation(), + findFittingAssignments(assignment, assignableNodeIds, remainingNodeMemory), + assignment.getMaxAssignedAllocations(), + Priority.LOW + ) + ) + .forEach(planModels::add); + if (modelToAdd.isPresent() && modelToAdd.get().getPriority() == Priority.LOW) { + StartTrainedModelDeploymentAction.TaskParams taskParams = modelToAdd.get(); + planModels.add( + new AssignmentPlan.Model( + taskParams.getModelId(), + taskParams.estimateMemoryUsageBytes(), + taskParams.getNumberOfAllocations(), + taskParams.getThreadsPerAllocation(), + Map.of(), + 0, + Priority.LOW + ) + ); + } + + logger.debug( + () -> format("Computing plan for low priority deployments. CPU cores fixed to [%s].", MAX_LOW_PRIORITY_MODELS_PER_NODE) ); - return new ZoneAwareAssignmentPlanner(nodesByZone, planModels).computePlan(); + + // No need to use the zone aware planner as there is only 1 allocation for low priority models. + return new AssignmentPlanner(planNodes, planModels).computePlan(); + } + + private Map findFittingAssignments( + TrainedModelAssignment assignment, + Set assignableNodeIds, + Map remainingNodeMemory + ) { + Map currentAssignments = assignment.getNodeRoutingTable() + .entrySet() + .stream() + // Filter out nodes that are no longer assignable + .filter(e -> assignableNodeIds.contains(e.getKey())) + .filter(e -> e.getValue().getState().isAnyOf(RoutingState.STARTING, RoutingState.STARTED, RoutingState.FAILED)) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getTargetAllocations())); + + final long modelMemoryBytes = assignment.getTaskParams().estimateMemoryUsageBytes(); + Map fittingAssignments = new HashMap<>(); + currentAssignments.entrySet().stream().filter(nodeToAllocations -> nodeToAllocations.getValue() > 0).forEach(nodeToAllocations -> { + if (remainingNodeMemory.get(nodeToAllocations.getKey()) >= modelMemoryBytes) { + fittingAssignments.put(nodeToAllocations.getKey(), nodeToAllocations.getValue()); + remainingNodeMemory.computeIfPresent(nodeToAllocations.getKey(), (k, v) -> v - modelMemoryBytes); + } + }); + return fittingAssignments; } private Map, List> createNodesByZoneMap() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AssignmentPlan.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AssignmentPlan.java index 8dd1abc48309e..8c3f33a34a525 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AssignmentPlan.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AssignmentPlan.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Tuple; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import java.util.ArrayList; import java.util.Collection; @@ -33,9 +34,21 @@ public record Model( int allocations, int threadsPerAllocation, Map currentAllocationsByNodeId, - int maxAssignedAllocations + int maxAssignedAllocations, + Priority priority ) { + public Model( + String id, + long memoryBytes, + int allocations, + int threadsPerAllocation, + Map currentAllocationsByNodeId, + int maxAssignedAllocations + ) { + this(id, memoryBytes, allocations, threadsPerAllocation, currentAllocationsByNodeId, maxAssignedAllocations, Priority.NORMAL); + } + int getCurrentAssignedAllocations() { return currentAllocationsByNodeId.values().stream().mapToInt(Integer::intValue).sum(); } @@ -238,7 +251,7 @@ public static Builder builder(Collection nodes, Collection models) return new Builder(nodes, models); } - static class Builder { + public static class Builder { private final Map> assignments; private final Map remainingNodeMemory; @@ -290,18 +303,19 @@ int getRemainingAllocations(Model m) { } boolean canAssign(Model model, Node node, int allocations) { - return (isAlreadyAssigned(model, node) || model.memoryBytes() <= remainingNodeMemory.get(node)) - && allocations * model.threadsPerAllocation() <= remainingNodeCores.get(node); + return (isAlreadyAssigned(model, node) + || (model.memoryBytes() <= remainingNodeMemory.get(node)) + && (model.priority == Priority.LOW || allocations * model.threadsPerAllocation() <= remainingNodeCores.get(node))); } - Builder assignModelToNode(Model model, Node node, int allocations) { + public Builder assignModelToNode(Model model, Node node, int allocations) { if (allocations <= 0) { return this; } if (isAlreadyAssigned(model, node) == false && model.memoryBytes() > remainingNodeMemory.get(node)) { throw new IllegalArgumentException("not enough memory on node [" + node.id() + "] to assign model [" + model.id() + "]"); } - if (allocations * model.threadsPerAllocation() > remainingNodeCores.get(node)) { + if (model.priority == Priority.NORMAL && allocations * model.threadsPerAllocation() > remainingNodeCores.get(node)) { throw new IllegalArgumentException( "not enough cores on node [" + node.id() @@ -318,7 +332,9 @@ Builder assignModelToNode(Model model, Node node, int allocations) { long additionalModelMemory = isAlreadyAssigned(model, node) ? 0 : model.memoryBytes; assignments.get(model).compute(node, (n, remAllocations) -> remAllocations + allocations); remainingNodeMemory.compute(node, (n, remMemory) -> remMemory - additionalModelMemory); - remainingNodeCores.compute(node, (n, remCores) -> remCores - allocations * model.threadsPerAllocation()); + if (model.priority == Priority.NORMAL) { + remainingNodeCores.compute(node, (n, remCores) -> remCores - allocations * model.threadsPerAllocation()); + } remainingModelAllocations.compute(model, (m, remModelThreads) -> remModelThreads - allocations); return this; } @@ -327,11 +343,11 @@ private boolean isAlreadyAssigned(Model model, Node node) { return model.currentAllocationsByNodeId().containsKey(node.id()) || assignments.get(model).get(node) > 0; } - void accountMemory(Model m, Node n) { + public void accountMemory(Model m, Node n) { remainingNodeMemory.computeIfPresent(n, (k, v) -> v - m.memoryBytes()); } - AssignmentPlan build() { + public AssignmentPlan build() { Map> finalAssignments = new HashMap<>(); for (Model m : assignments.keySet()) { Map allocationsPerNode = new HashMap<>(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AssignmentPlanner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AssignmentPlanner.java index df9ecd6a5de38..3461abe73d5bb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AssignmentPlanner.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AssignmentPlanner.java @@ -42,14 +42,14 @@ * attempt to find a solution that provides at least one allocation to * previously assigned models. */ -class AssignmentPlanner { +public class AssignmentPlanner { private static final Logger logger = LogManager.getLogger(AssignmentPlanner.class); private final List nodes; private final List models; - AssignmentPlanner(List nodes, List models) { + public AssignmentPlanner(List nodes, List models) { this.nodes = nodes.stream().sorted(Comparator.comparing(Node::id)).toList(); this.models = models.stream().sorted(Comparator.comparing(Model::id)).toList(); } @@ -107,7 +107,7 @@ private AssignmentPlan solveSatisfyingCurrentAssignments() { } private AssignmentPlan solveAllocatingAtLeastOnceModelsThatWerePreviouslyAllocated() { - logger.debug(() -> "Attempting to solve assigning at least one allocations to previously assigned models"); + logger.debug(() -> "Attempting to solve assigning at least one allocation to previously assigned models"); List previouslyAssignedModelsOnly = models.stream() .filter(m -> m.hasEverBeenAllocated()) .map( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/ZoneAwareAssignmentPlanner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/ZoneAwareAssignmentPlanner.java index c1df5d4d7e933..c198e858ddf9d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/ZoneAwareAssignmentPlanner.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/planning/ZoneAwareAssignmentPlanner.java @@ -171,17 +171,16 @@ private AssignmentPlan swapOriginalModelsInPlan(AssignmentPlan plan, List final Map originalNodeById = allNodes.stream().collect(Collectors.toMap(Node::id, Function.identity())); AssignmentPlan.Builder planBuilder = AssignmentPlan.builder(allNodes, models); for (Model m : planModels) { - Optional> nodeAssignments = plan.assignments(m); - if (nodeAssignments.isPresent()) { - nodeAssignments.get() - .entrySet() - .forEach( - e -> planBuilder.assignModelToNode( - originalModelById.get(m.id()), - originalNodeById.get(e.getKey().id()), - e.getValue() - ) - ); + Model originalModel = originalModelById.get(m.id()); + Map nodeAssignments = plan.assignments(m).orElse(Map.of()); + for (Map.Entry assignment : nodeAssignments.entrySet()) { + Node originalNode = originalNodeById.get(assignment.getKey().id()); + planBuilder.assignModelToNode(originalModel, originalNode, assignment.getValue()); + if (originalModel.currentAllocationsByNodeId().containsKey(originalNode.id())) { + // As the node has all its available memory we need to manually account memory of models with + // current allocations. + planBuilder.accountMemory(m, originalNode); + } } } return planBuilder.build(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTask.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTask.java index 1812814206086..05f03d8d393fd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTask.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTask.java @@ -80,7 +80,8 @@ public void updateNumberOfAllocations(int numberOfAllocations) { numberOfAllocations, params.getThreadsPerAllocation(), params.getQueueCapacity(), - params.getCacheSize().orElse(null) + params.getCacheSize().orElse(null), + params.getPriority() ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java index 2c440941b5224..2ba81386fa754 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java @@ -155,6 +155,7 @@ private void doSearch( // this many docs so far. int lastNum = numDocsWritten - 1; for (SearchHit hit : searchResponse.getHits().getHits()) { + logger.debug(() -> format("[%s] Restoring model definition doc with id [%s]", modelId, hit.getId())); try { TrainedModelDefinitionDoc doc = parseModelDefinitionDocLenientlyFromSource( hit.getSourceRef(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/NativePyTorchProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/NativePyTorchProcessFactory.java index 899e5f6b7fc8b..64c466ee610fa 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/NativePyTorchProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/NativePyTorchProcessFactory.java @@ -99,13 +99,7 @@ public NativePyTorchProcess createProcess( } private void executeProcess(ProcessPipes processPipes, TrainedModelDeploymentTask task) { - PyTorchBuilder pyTorchBuilder = new PyTorchBuilder( - nativeController, - processPipes, - task.getParams().getThreadsPerAllocation(), - task.getParams().getNumberOfAllocations(), - task.getParams().getCacheSizeBytes() - ); + PyTorchBuilder pyTorchBuilder = new PyTorchBuilder(nativeController, processPipes, task.getParams()); try { pyTorchBuilder.build(); } catch (InterruptedException e) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchBuilder.java index 1e9cdc64ccc2b..76fe9336fa405 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchBuilder.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.ml.inference.pytorch.process; +import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.ml.process.NativeController; import org.elasticsearch.xpack.ml.process.ProcessPipes; @@ -24,25 +26,20 @@ public class PyTorchBuilder { private static final String NUM_THREADS_PER_ALLOCATION_ARG = "--numThreadsPerAllocation="; private static final String NUM_ALLOCATIONS_ARG = "--numAllocations="; private static final String CACHE_MEMORY_LIMIT_BYTES_ARG = "--cacheMemorylimitBytes="; + private static final String LOW_PRIORITY_ARG = "--lowPriority"; private final NativeController nativeController; private final ProcessPipes processPipes; - private final int threadsPerAllocation; - private final int numberOfAllocations; - private final long cacheMemoryLimitBytes; + private final StartTrainedModelDeploymentAction.TaskParams taskParams; public PyTorchBuilder( NativeController nativeController, ProcessPipes processPipes, - int threadPerAllocation, - int numberOfAllocations, - long cacheMemoryLimitBytes + StartTrainedModelDeploymentAction.TaskParams taskParams ) { this.nativeController = Objects.requireNonNull(nativeController); this.processPipes = Objects.requireNonNull(processPipes); - this.threadsPerAllocation = threadPerAllocation; - this.numberOfAllocations = numberOfAllocations; - this.cacheMemoryLimitBytes = cacheMemoryLimitBytes; + this.taskParams = Objects.requireNonNull(taskParams); } public void build() throws IOException, InterruptedException { @@ -58,10 +55,13 @@ private List buildCommand() { // License was validated when the trained model was started command.add(LICENSE_KEY_VALIDATED_ARG + true); - command.add(NUM_THREADS_PER_ALLOCATION_ARG + threadsPerAllocation); - command.add(NUM_ALLOCATIONS_ARG + numberOfAllocations); - if (cacheMemoryLimitBytes > 0) { - command.add(CACHE_MEMORY_LIMIT_BYTES_ARG + cacheMemoryLimitBytes); + command.add(NUM_THREADS_PER_ALLOCATION_ARG + taskParams.getThreadsPerAllocation()); + command.add(NUM_ALLOCATIONS_ARG + taskParams.getNumberOfAllocations()); + if (taskParams.getCacheSizeBytes() > 0) { + command.add(CACHE_MEMORY_LIMIT_BYTES_ARG + taskParams.getCacheSizeBytes()); + } + if (taskParams.getPriority() == Priority.LOW) { + command.add(LOW_PRIORITY_ARG); } return command; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java index 424cd4d3ee16a..436efa05488ce 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java @@ -92,6 +92,12 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient ); } request.setQueueCapacity(restRequest.paramAsInt(QUEUE_CAPACITY.getPreferredName(), request.getQueueCapacity())); + request.setPriority( + restRequest.param( + StartTrainedModelDeploymentAction.TaskParams.PRIORITY.getPreferredName(), + request.getPriority().toString() + ) + ); } return channel -> client.execute(StartTrainedModelDeploymentAction.INSTANCE, request, new RestToXContentListener<>(channel)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java index dffcd96cea867..4ae21b58ed332 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java @@ -57,6 +57,7 @@ import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.NerConfig; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig; @@ -346,7 +347,7 @@ public void testUsage() throws Exception { ), 3, null, - new AssignmentStats("model_3", null, null, null, null, Instant.now(), List.of()).setState( + new AssignmentStats("model_3", null, null, null, null, Instant.now(), List.of(), Priority.NORMAL).setState( AssignmentState.STOPPING ) ), @@ -415,7 +416,8 @@ public void testUsage() throws Exception { 34.0, 1L ) - ) + ), + Priority.NORMAL ).setState(AssignmentState.STARTED).setAllocationStatus(new AllocationStatus(2, 2)) ) ), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsActionTests.java index d4c63421bc903..3d5edc5cfa0ca 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStatsTests; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; @@ -85,7 +86,8 @@ public void testAddFailedRoutes_GivenMixedResponses() throws UnknownHostExceptio randomBoolean() ? null : randomIntBetween(1, 10000), randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, 1000000)), Instant.now(), - nodeStatsList + nodeStatsList, + randomFrom(Priority.values()) ); Map> badRoutes = new HashMap<>(); @@ -121,7 +123,8 @@ public void testAddFailedRoutes_TaskResultIsOverwritten() throws UnknownHostExce randomBoolean() ? null : randomIntBetween(1, 10000), randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, 1000000)), Instant.now(), - nodeStatsList + nodeStatsList, + randomFrom(Priority.values()) ); var response = new GetDeploymentStatsAction.Response(Collections.emptyList(), Collections.emptyList(), List.of(model1), 1); @@ -154,7 +157,7 @@ private DiscoveryNodes buildNodes(String... nodeIds) throws UnknownHostException private static TrainedModelAssignment createAssignment(String modelId) { return TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId, 1024, 1, 1, 1, ByteSizeValue.ofBytes(1024)) + new StartTrainedModelDeploymentAction.TaskParams(modelId, 1024, 1, 1, 1, ByteSizeValue.ofBytes(1024), Priority.NORMAL) ).build(); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java index c7a8028d80ce3..326044e05c24c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; @@ -1053,10 +1054,10 @@ public void testCpuModelAssignmentRequirements() { MlMemoryAutoscalingDecider.modelAssignmentsRequireMoreThanHalfCpu( List.of( TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 2, 3, 100, null) + new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 2, 3, 100, null, Priority.NORMAL) ).build(), TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 1, 100, null) + new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 1, 100, null, Priority.NORMAL) ).build() ), withMlNodes("ml_node_1", "ml_node_2") @@ -1066,10 +1067,10 @@ public void testCpuModelAssignmentRequirements() { MlMemoryAutoscalingDecider.modelAssignmentsRequireMoreThanHalfCpu( List.of( TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 3, 100, null) + new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 3, 100, null, Priority.NORMAL) ).build(), TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 1, 100, null) + new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 1, 100, null, Priority.NORMAL) ).build() ), withMlNodes("ml_node_1", "ml_node_2") @@ -1079,10 +1080,10 @@ public void testCpuModelAssignmentRequirements() { MlMemoryAutoscalingDecider.modelAssignmentsRequireMoreThanHalfCpu( List.of( TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 3, 100, null) + new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 3, 100, null, Priority.NORMAL) ).build(), TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 1, 100, null) + new StartTrainedModelDeploymentAction.TaskParams("model1", TEST_JOB_SIZE, 1, 1, 100, null, Priority.NORMAL) ).build() ), withMlNodes("ml_node_1", "ml_node_2", "ml_node_3", "ml_node_4") diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlProcessorAutoscalingDeciderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlProcessorAutoscalingDeciderTests.java index 1e246a0583053..72a500042091f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlProcessorAutoscalingDeciderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlProcessorAutoscalingDeciderTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderContext; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; @@ -67,13 +68,29 @@ public void testScale_GivenCurrentCapacityIsUsedExactly() { .addNewAssignment( modelId1, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId1, 42L, 2, 3, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId1, + 42L, + 2, + 3, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ).addRoutingEntry(mlNodeId1, new RoutingInfo(2, 2, RoutingState.STARTED, "")) ) .addNewAssignment( modelId2, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId2, 42L, 10, 1, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId2, + 42L, + 10, + 1, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ) .addRoutingEntry(mlNodeId1, new RoutingInfo(2, 2, RoutingState.STARTED, "")) .addRoutingEntry(mlNodeId2, new RoutingInfo(8, 8, RoutingState.STARTED, "")) @@ -118,13 +135,29 @@ public void testScale_GivenUnsatisfiedDeployments() { .addNewAssignment( modelId1, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId1, 42L, 1, 8, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId1, + 42L, + 1, + 8, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ) ) .addNewAssignment( modelId2, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId2, 42L, 3, 4, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId2, + 42L, + 3, + 4, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ) .addRoutingEntry(mlNodeId1, new RoutingInfo(1, 1, RoutingState.STARTED, "")) .addRoutingEntry(mlNodeId2, new RoutingInfo(1, 1, RoutingState.STARTED, "")) @@ -148,6 +181,73 @@ public void testScale_GivenUnsatisfiedDeployments() { assertThat(capacity.reason(), equalTo("requesting scale up as there are unsatisfied deployments")); } + public void testScale_GivenUnsatisfiedDeploymentIsLowPriority_ShouldNotScaleUp() { + String modelId1 = "model-id-1"; + String modelId2 = "model-id-2"; + + String mlNodeId1 = "ml-node-id-1"; + String mlNodeId2 = "ml-node-id-2"; + String dataNodeId = "data-node-id"; + DiscoveryNode mlNode1 = buildNode(mlNodeId1, true, 4); + DiscoveryNode mlNode2 = buildNode(mlNodeId2, true, 4); + DiscoveryNode dataNode = buildNode(dataNodeId, false, 24); + + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .nodes(DiscoveryNodes.builder().add(mlNode1).add(mlNode2).add(dataNode).build()) + .metadata( + Metadata.builder() + .putCustom( + TrainedModelAssignmentMetadata.NAME, + TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment( + modelId1, + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + modelId1, + 42L, + 1, + 1, + 1024, + ByteSizeValue.ONE, + Priority.LOW + ) + ) + ) + .addNewAssignment( + modelId2, + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + modelId2, + 42L, + 2, + 4, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) + ) + .addRoutingEntry(mlNodeId1, new RoutingInfo(1, 1, RoutingState.STARTED, "")) + .addRoutingEntry(mlNodeId2, new RoutingInfo(1, 1, RoutingState.STARTED, "")) + ) + .build() + ) + .build() + ) + .build(); + + MlProcessorAutoscalingDecider decider = newDecider(); + + MlProcessorAutoscalingCapacity capacity = decider.scale( + Settings.EMPTY, + newContext(clusterState), + new MlAutoscalingContext(clusterState) + ); + + assertThat(capacity.nodeProcessors(), equalTo(Processors.of(4.0))); + assertThat(capacity.tierProcessors(), equalTo(Processors.of(8.0))); + assertThat(capacity.reason(), equalTo("passing currently perceived capacity as it is fully used")); + } + public void testScale_GivenMoreThanHalfProcessorsAreUsed() { String modelId1 = "model-id-1"; String modelId2 = "model-id-2"; @@ -169,13 +269,29 @@ public void testScale_GivenMoreThanHalfProcessorsAreUsed() { .addNewAssignment( modelId1, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId1, 42L, 2, 2, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId1, + 42L, + 2, + 2, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ).addRoutingEntry(mlNodeId1, new RoutingInfo(2, 2, RoutingState.STARTED, "")) ) .addNewAssignment( modelId2, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId2, 42L, 1, 1, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId2, + 42L, + 1, + 1, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ).addRoutingEntry(mlNodeId2, new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .build() @@ -221,13 +337,29 @@ public void testScale_GivenDownScalePossible_DelayNotSatisfied() { .addNewAssignment( modelId1, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId1, 42L, 2, 2, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId1, + 42L, + 2, + 2, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ).addRoutingEntry(mlNodeId1, new RoutingInfo(2, 2, RoutingState.STARTED, "")) ) .addNewAssignment( modelId2, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId2, 42L, 1, 1, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId2, + 42L, + 1, + 1, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ).addRoutingEntry(mlNodeId2, new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .build() @@ -271,13 +403,29 @@ public void testScale_GivenDownScalePossible_DelaySatisfied() { .addNewAssignment( modelId1, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId1, 42L, 2, 2, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId1, + 42L, + 2, + 2, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ).addRoutingEntry(mlNodeId1, new RoutingInfo(2, 2, RoutingState.STARTED, "")) ) .addNewAssignment( modelId2, TrainedModelAssignment.Builder.empty( - new StartTrainedModelDeploymentAction.TaskParams(modelId2, 42L, 1, 1, 1024, ByteSizeValue.ONE) + new StartTrainedModelDeploymentAction.TaskParams( + modelId2, + 42L, + 1, + 1, + 1024, + ByteSizeValue.ONE, + Priority.NORMAL + ) ).addRoutingEntry(mlNodeId2, new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .build() @@ -304,6 +452,76 @@ public void testScale_GivenDownScalePossible_DelaySatisfied() { assertThat(capacity.reason(), containsString("requesting scale down as tier and/or node size could be smaller")); } + public void testScale_GivenLowPriorityDeploymentsOnly() { + String modelId1 = "model-id-1"; + String modelId2 = "model-id-2"; + + String mlNodeId1 = "ml-node-id-1"; + String mlNodeId2 = "ml-node-id-2"; + String dataNodeId = "data-node-id"; + DiscoveryNode mlNode1 = buildNode(mlNodeId1, true, 4); + DiscoveryNode mlNode2 = buildNode(mlNodeId2, true, 4); + DiscoveryNode dataNode = buildNode(dataNodeId, false, 24); + + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .nodes(DiscoveryNodes.builder().add(mlNode1).add(mlNode2).add(dataNode).build()) + .metadata( + Metadata.builder() + .putCustom( + TrainedModelAssignmentMetadata.NAME, + TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment( + modelId1, + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + modelId1, + 42L, + 1, + 1, + 1024, + ByteSizeValue.ONE, + Priority.LOW + ) + ).addRoutingEntry(mlNodeId1, new RoutingInfo(1, 1, RoutingState.STARTED, "")) + ) + .addNewAssignment( + modelId2, + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + modelId2, + 42L, + 1, + 1, + 1024, + ByteSizeValue.ONE, + Priority.LOW + ) + ).addRoutingEntry(mlNodeId1, new RoutingInfo(1, 1, RoutingState.STARTED, "")) + ) + .build() + ) + .build() + ) + .build(); + + TimeMachine timeMachine = new TimeMachine(); + scaleTimer = new ScaleTimer(timeMachine); + MlProcessorAutoscalingDecider decider = newDecider(); + scaleTimer.markScale(); + scaleTimer.markDownScaleAndGetMillisLeftFromDelay(Settings.EMPTY); + timeMachine.setOffset(TimeValue.timeValueHours(1)); + + MlProcessorAutoscalingCapacity capacity = decider.scale( + Settings.EMPTY, + newContext(clusterState), + new MlAutoscalingContext(clusterState) + ); + + assertThat(capacity.nodeProcessors(), equalTo(Processors.ZERO)); + assertThat(capacity.tierProcessors(), equalTo(Processors.of(0.1))); + assertThat(capacity.reason(), equalTo("requesting scale down as tier and/or node size could be smaller")); + } + private static DiscoveryNode buildNode(String name, boolean isML, double allocatedProcessors) { return new DiscoveryNode( name, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java index f1b0e04ccceee..4fd2ca42fde47 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelAssignmentRoutingInfoAction; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfoUpdate; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; @@ -1505,7 +1506,8 @@ private static StartTrainedModelDeploymentAction.TaskParams newParams( numberOfAllocations, threadsPerAllocation, 1024, - ByteSizeValue.ofBytes(modelSize) + ByteSizeValue.ofBytes(modelSize), + Priority.NORMAL ); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadataTests.java index 8e42febcb6993..d2a5f89350ac7 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadataTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadataTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentTests; @@ -66,7 +67,8 @@ private static StartTrainedModelDeploymentAction.TaskParams randomParams(String randomIntBetween(1, 8), randomIntBetween(1, 8), randomIntBetween(1, 10000), - randomBoolean() ? null : ByteSizeValue.ofBytes(randomNonNegativeLong()) + randomBoolean() ? null : ByteSizeValue.ofBytes(randomNonNegativeLong()), + randomFrom(Priority.values()) ); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeServiceTests.java index 4a7fbd2908ea6..b2053eed5be62 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeServiceTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelAssignmentRoutingInfoAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; @@ -631,7 +632,8 @@ private static StartTrainedModelDeploymentAction.TaskParams newParams(String mod 1, 1, 1024, - randomBoolean() ? null : ByteSizeValue.ofBytes(randomNonNegativeLong()) + randomBoolean() ? null : ByteSizeValue.ofBytes(randomNonNegativeLong()), + randomFrom(Priority.values()) ); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentRebalancerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentRebalancerTests.java index 10bb7bb8b69d3..645e030ba2ff1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentRebalancerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentRebalancerTests.java @@ -16,12 +16,14 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.NodeLoad; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -29,6 +31,7 @@ import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; @@ -50,8 +53,8 @@ public void testRebalance_GivenNoAssignments() throws Exception { public void testRebalance_GivenAllAssignmentsAreSatisfied_ShouldMakeNoChanges() throws Exception { String modelId1 = "model-1"; String modelId2 = "model-2"; - StartTrainedModelDeploymentAction.TaskParams taskParams1 = newParams(modelId1, 1024L, 1, 2); - StartTrainedModelDeploymentAction.TaskParams taskParams2 = newParams(modelId2, 1024L, 4, 1); + StartTrainedModelDeploymentAction.TaskParams taskParams1 = normalPriorityParams(modelId1, 1024L, 1, 2); + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, 1024L, 4, 1); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() .addNewAssignment( modelId1, @@ -83,8 +86,8 @@ public void testRebalance_GivenAllAssignmentsAreSatisfied_GivenOutdatedRoutingEn String modelId1 = "model-1"; String modelId2 = "model-2"; - StartTrainedModelDeploymentAction.TaskParams taskParams1 = newParams(modelId1, 1024L, 1, 2); - StartTrainedModelDeploymentAction.TaskParams taskParams2 = newParams(modelId2, 1024L, 4, 1); + StartTrainedModelDeploymentAction.TaskParams taskParams1 = normalPriorityParams(modelId1, 1024L, 1, 2); + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, 1024L, 4, 1); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() .addNewAssignment( modelId1, @@ -123,7 +126,7 @@ public void testRebalance_GivenAllAssignmentsAreSatisfied_GivenOutdatedRoutingEn public void testRebalance_GivenModelToAddAlreadyExists() { String modelId = "model-to-add"; - StartTrainedModelDeploymentAction.TaskParams taskParams = newParams(modelId, 1024L, 1, 1); + StartTrainedModelDeploymentAction.TaskParams taskParams = normalPriorityParams(modelId, 1024L, 1, 1); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() .addNewAssignment(modelId, TrainedModelAssignment.Builder.empty(taskParams)) .build(); @@ -135,7 +138,7 @@ public void testRebalance_GivenModelToAddAlreadyExists() { public void testRebalance_GivenFirstModelToAdd_NoMLNodes() throws Exception { String modelId = "model-to-add"; - StartTrainedModelDeploymentAction.TaskParams taskParams = newParams(modelId, 1024L, 1, 1); + StartTrainedModelDeploymentAction.TaskParams taskParams = normalPriorityParams(modelId, 1024L, 1, 1); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty().build(); TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( @@ -158,7 +161,7 @@ public void testRebalance_GivenFirstModelToAdd_NotEnoughProcessors() throws Exce DiscoveryNode node = buildNode("node-1", nodeMemoryBytes, 3); String modelId = "model-to-add"; - StartTrainedModelDeploymentAction.TaskParams taskParams = newParams(modelId, 1024L, 1, 4); + StartTrainedModelDeploymentAction.TaskParams taskParams = normalPriorityParams(modelId, 1024L, 1, 4); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty().build(); Map nodeLoads = new HashMap<>(); @@ -187,7 +190,7 @@ public void testRebalance_GivenFirstModelToAdd_NotEnoughProcessors() throws Exce public void testRebalance_GivenFirstModelToAdd_NotEnoughMemory() throws Exception { String modelId = "model-to-add"; - StartTrainedModelDeploymentAction.TaskParams taskParams = newParams(modelId, ByteSizeValue.ofGb(2).getBytes(), 1, 1); + StartTrainedModelDeploymentAction.TaskParams taskParams = normalPriorityParams(modelId, ByteSizeValue.ofGb(2).getBytes(), 1, 1); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty().build(); Map nodeLoads = new HashMap<>(); long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); @@ -213,7 +216,7 @@ public void testRebalance_GivenFirstModelToAdd_NotEnoughMemory() throws Exceptio public void testRebalance_GivenFirstModelToAdd_ErrorDetectingNodeLoad() throws Exception { String modelId = "model-to-add"; - StartTrainedModelDeploymentAction.TaskParams taskParams = newParams(modelId, ByteSizeValue.ofGb(2).getBytes(), 1, 1); + StartTrainedModelDeploymentAction.TaskParams taskParams = normalPriorityParams(modelId, ByteSizeValue.ofGb(2).getBytes(), 1, 1); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty().build(); Map nodeLoads = new HashMap<>(); long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); @@ -245,7 +248,7 @@ public void testRebalance_GivenProblemsOnMultipleNodes() throws Exception { DiscoveryNode node2 = buildNode("node-2", ByteSizeValue.ofGb(10).getBytes(), 3); String modelId = "model-to-add"; - StartTrainedModelDeploymentAction.TaskParams taskParams = newParams(modelId, ByteSizeValue.ofGb(2).getBytes(), 1, 4); + StartTrainedModelDeploymentAction.TaskParams taskParams = normalPriorityParams(modelId, ByteSizeValue.ofGb(2).getBytes(), 1, 4); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty().build(); Map nodeLoads = new HashMap<>(); nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(ByteSizeValue.ofGb(1).getBytes()).build()); @@ -278,7 +281,7 @@ public void testRebalance_GivenFirstModelToAdd_FitsFully() throws Exception { DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 4); String modelId = "model-to-add"; - StartTrainedModelDeploymentAction.TaskParams taskParams = newParams(modelId, 1024L, 1, 1); + StartTrainedModelDeploymentAction.TaskParams taskParams = normalPriorityParams(modelId, 1024L, 1, 1); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty().build(); Map nodeLoads = new HashMap<>(); nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); @@ -308,11 +311,11 @@ public void testRebalance_GivenModelToAdd_AndPreviousAssignments_AndTwoNodes_All String modelToAddId = "model-to-add"; String previousModelId = "previous-model"; - StartTrainedModelDeploymentAction.TaskParams taskParams = newParams(modelToAddId, 1024L, 1, 2); + StartTrainedModelDeploymentAction.TaskParams taskParams = normalPriorityParams(modelToAddId, 1024L, 1, 2); TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() .addNewAssignment( previousModelId, - TrainedModelAssignment.Builder.empty(newParams(previousModelId, 1024L, 3, 2)) + TrainedModelAssignment.Builder.empty(normalPriorityParams(previousModelId, 1024L, 3, 2)) .addRoutingEntry("node-1", new RoutingInfo(2, 2, RoutingState.STARTED, "")) .addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) @@ -369,13 +372,13 @@ public void testRebalance_GivenPreviousAssignments_AndNewNode() throws Exception TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() .addNewAssignment( previousModel1Id, - TrainedModelAssignment.Builder.empty(newParams(previousModel1Id, 1024L, 3, 2)) + TrainedModelAssignment.Builder.empty(normalPriorityParams(previousModel1Id, 1024L, 3, 2)) .addRoutingEntry("node-1", new RoutingInfo(2, 2, RoutingState.STARTED, "")) .addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .addNewAssignment( previousModel2Id, - TrainedModelAssignment.Builder.empty(newParams(previousModel2Id, 1024L, 4, 1)) + TrainedModelAssignment.Builder.empty(normalPriorityParams(previousModel2Id, 1024L, 4, 1)) .addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .build(); @@ -434,13 +437,13 @@ public void testRebalance_GivenPreviousAssignments_AndRemovedNode_AndRemainingNo TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() .addNewAssignment( previousModel1Id, - TrainedModelAssignment.Builder.empty(newParams(previousModel1Id, 1024L, 3, 2)) + TrainedModelAssignment.Builder.empty(normalPriorityParams(previousModel1Id, 1024L, 3, 2)) .addRoutingEntry("node-1", new RoutingInfo(2, 2, RoutingState.STARTED, "")) .addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .addNewAssignment( previousModel2Id, - TrainedModelAssignment.Builder.empty(newParams(previousModel2Id, 1024L, 4, 1)) + TrainedModelAssignment.Builder.empty(normalPriorityParams(previousModel2Id, 1024L, 4, 1)) .addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .build(); @@ -503,13 +506,13 @@ public void testRebalance_GivenPreviousAssignments_AndRemovedNode_AndRemainingNo TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() .addNewAssignment( previousModel1Id, - TrainedModelAssignment.Builder.empty(newParams(previousModel1Id, 1024L, 3, 2)) + TrainedModelAssignment.Builder.empty(normalPriorityParams(previousModel1Id, 1024L, 3, 2)) .addRoutingEntry("node-1", new RoutingInfo(2, 2, RoutingState.STARTED, "")) .addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .addNewAssignment( previousModel2Id, - TrainedModelAssignment.Builder.empty(newParams(previousModel2Id, 1024L, 1, 1)) + TrainedModelAssignment.Builder.empty(normalPriorityParams(previousModel2Id, 1024L, 1, 1)) .addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")) ) .build(); @@ -557,7 +560,7 @@ public void testRebalance_GivenFailedAssignment_RestartsAssignment() throws Exce TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() .addNewAssignment( modelId, - TrainedModelAssignment.Builder.empty(newParams(modelId, 1024L, 1, 1)) + TrainedModelAssignment.Builder.empty(normalPriorityParams(modelId, 1024L, 1, 1)) .addRoutingEntry("node-1", new RoutingInfo(1, 1, RoutingState.FAILED, "some error")) ) .build(); @@ -584,7 +587,441 @@ public void testRebalance_GivenFailedAssignment_RestartsAssignment() throws Exce assertThat(assignment.getReason().isPresent(), is(false)); } - private static StartTrainedModelDeploymentAction.TaskParams newParams( + public void testRebalance_GivenLowPriorityModelToAdd_OnlyModel_NotEnoughMemory() throws Exception { + String modelId = "model-to-add"; + StartTrainedModelDeploymentAction.TaskParams taskParams = lowPriorityParams(modelId, ByteSizeValue.ofGb(2).getBytes()); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty().build(); + Map nodeLoads = new HashMap<>(); + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + nodeLoads.put(buildNode("node-1", nodeMemoryBytes, 3), NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(), + Optional.of(taskParams) + ).rebalance().build(); + + TrainedModelAssignment assignment = result.getModelAssignment(modelId); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(anEmptyMap())); + assertThat(assignment.getReason().isPresent(), is(true)); + assertThat( + assignment.getReason().get(), + containsString("Could not assign (more) allocations on node [node-1]. Reason: This node has insufficient available memory.") + ); + } + + public void testRebalance_GivenLowPriorityModelToAdd_NotEnoughMemoryNorProcessors() throws Exception { + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 1); + DiscoveryNode node2 = buildNode("node-2", nodeMemoryBytes, 1); + + Map nodeLoads = new HashMap<>(); + nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + nodeLoads.put(node2, NodeLoad.builder("node-2").setMaxMemory(nodeMemoryBytes).build()); + + String modelId1 = "model-1"; + StartTrainedModelDeploymentAction.TaskParams taskParams1 = lowPriorityParams(modelId1, ByteSizeValue.ofMb(300).getBytes()); + String modelId2 = "model-2"; + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, ByteSizeValue.ofMb(300).getBytes(), 2, 1); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment( + modelId2, + TrainedModelAssignment.Builder.empty(taskParams2) + .addRoutingEntry("node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")) + .addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")) + ) + .build(); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(List.of("zone-1"), List.of(node1), List.of("zone-2"), List.of(node2)), + Optional.of(taskParams1) + ).rebalance().build(); + + TrainedModelAssignment assignment = result.getModelAssignment(modelId1); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(anEmptyMap())); + assertThat(assignment.getReason().isPresent(), is(true)); + assertThat( + assignment.getReason().get(), + containsString("Could not assign (more) allocations on node [node-1]. Reason: This node has insufficient available memory.") + ); + assertThat( + assignment.getReason().get(), + containsString("Could not assign (more) allocations on node [node-2]. Reason: This node has insufficient available memory.") + ); + } + + public void testRebalance_GivenMixedPriorityModels_NotEnoughMemoryForLowPriority() throws Exception { + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 7); + + Map nodeLoads = new HashMap<>(); + nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + + String modelId1 = "model-1"; + StartTrainedModelDeploymentAction.TaskParams taskParams1 = lowPriorityParams(modelId1, ByteSizeValue.ofMb(250).getBytes()); + String modelId2 = "model-2"; + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, ByteSizeValue.ofMb(300).getBytes(), 1, 1); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment(modelId1, TrainedModelAssignment.Builder.empty(taskParams1)) + .addNewAssignment(modelId2, TrainedModelAssignment.Builder.empty(taskParams2)) + .build(); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(List.of(), List.of(node1)), + Optional.empty() + ).rebalance().build(); + + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId1); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(anEmptyMap())); + assertThat(assignment.getReason().isPresent(), is(true)); + assertThat( + assignment.getReason().get(), + containsString("Could not assign (more) allocations on node [node-1]. Reason: This node has insufficient available memory.") + ); + } + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId2); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-1")); + assertThat(assignment.getNodeRoutingTable().get("node-1").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + } + + public void testRebalance_GivenMixedPriorityModels_TwoZones_EachNodeCanHoldOneModel() throws Exception { + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 1); + DiscoveryNode node2 = buildNode("node-2", nodeMemoryBytes, 1); + + Map nodeLoads = new HashMap<>(); + nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + nodeLoads.put(node2, NodeLoad.builder("node-2").setMaxMemory(nodeMemoryBytes).build()); + + String modelId1 = "model-1"; + StartTrainedModelDeploymentAction.TaskParams taskParams1 = lowPriorityParams(modelId1, ByteSizeValue.ofMb(300).getBytes()); + String modelId2 = "model-2"; + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, ByteSizeValue.ofMb(300).getBytes(), 1, 1); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment(modelId1, TrainedModelAssignment.Builder.empty(taskParams1)) + .addNewAssignment( + modelId2, + TrainedModelAssignment.Builder.empty(taskParams2).addRoutingEntry("node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")) + ) + .build(); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(List.of("zone-1"), List.of(node1), List.of("zone-2"), List.of(node2)), + Optional.empty() + ).rebalance().build(); + + List assignedNodes = new ArrayList<>(); + + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId1); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + String assignedNode = assignment.getNodeRoutingTable().keySet().iterator().next(); + assertThat(assignment.getNodeRoutingTable().get(assignedNode).getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get(assignedNode).getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get(assignedNode).getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + assignedNodes.add(assignedNode); + } + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId2); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTED)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + String assignedNode = assignment.getNodeRoutingTable().keySet().iterator().next(); + assertThat(assignment.getNodeRoutingTable().get(assignedNode).getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get(assignedNode).getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get(assignedNode).getState(), equalTo(RoutingState.STARTED)); + assertThat(assignment.getReason().isPresent(), is(false)); + assignedNodes.add(assignedNode); + } + + assertThat(assignedNodes, containsInAnyOrder("node-1", "node-2")); + } + + public void testRebalance_GivenModelUsingAllCpu_FittingLowPriorityModelCanStart() throws Exception { + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 7); + + Map nodeLoads = new HashMap<>(); + nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + + String modelId1 = "model-1"; + StartTrainedModelDeploymentAction.TaskParams taskParams1 = lowPriorityParams(modelId1, ByteSizeValue.ofMb(250).getBytes()); + String modelId2 = "model-2"; + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, ByteSizeValue.ofMb(300).getBytes(), 1, 1); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment(modelId1, TrainedModelAssignment.Builder.empty(taskParams1)) + .addNewAssignment(modelId2, TrainedModelAssignment.Builder.empty(taskParams2)) + .build(); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(List.of(), List.of(node1)), + Optional.empty() + ).rebalance().build(); + + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId1); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(anEmptyMap())); + assertThat(assignment.getReason().isPresent(), is(true)); + assertThat( + assignment.getReason().get(), + containsString("Could not assign (more) allocations on node [node-1]. Reason: This node has insufficient available memory.") + ); + } + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId2); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-1")); + assertThat(assignment.getNodeRoutingTable().get("node-1").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + } + + public void testRebalance_GivenMultipleLowPriorityModels_AndMultipleNodes() throws Exception { + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 1); + DiscoveryNode node2 = buildNode("node-2", nodeMemoryBytes, 1); + + Map nodeLoads = new HashMap<>(); + nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + nodeLoads.put(node2, NodeLoad.builder("node-2").setMaxMemory(nodeMemoryBytes).build()); + + String modelId1 = "model-1"; + StartTrainedModelDeploymentAction.TaskParams taskParams1 = lowPriorityParams(modelId1, ByteSizeValue.ofMb(100).getBytes()); + String modelId2 = "model-2"; + StartTrainedModelDeploymentAction.TaskParams taskParams2 = lowPriorityParams(modelId2, ByteSizeValue.ofMb(100).getBytes()); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment(modelId1, TrainedModelAssignment.Builder.empty(taskParams1)) + .addNewAssignment(modelId2, TrainedModelAssignment.Builder.empty(taskParams2)) + .build(); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(List.of(), List.of(node1, node2)), + Optional.empty() + ).rebalance().build(); + + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId1); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-1")); + assertThat(assignment.getNodeRoutingTable().get("node-1").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId2); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-1")); + assertThat(assignment.getNodeRoutingTable().get("node-1").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + } + + public void testRebalance_GivenNormalPriorityModelToLoad_EvictsLowPriorityModel() throws Exception { + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 1); + + Map nodeLoads = new HashMap<>(); + nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + + String modelId1 = "model-1"; + StartTrainedModelDeploymentAction.TaskParams taskParams1 = lowPriorityParams(modelId1, ByteSizeValue.ofMb(300).getBytes()); + String modelId2 = "model-2"; + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, ByteSizeValue.ofMb(300).getBytes(), 1, 1); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment( + modelId1, + TrainedModelAssignment.Builder.empty(taskParams1).addRoutingEntry("node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")) + ) + .build(); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(List.of(), List.of(node1)), + Optional.of(taskParams2) + ).rebalance().build(); + + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId1); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(anEmptyMap())); + assertThat(assignment.getReason().isPresent(), is(true)); + assertThat( + assignment.getReason().get(), + containsString("Could not assign (more) allocations on node [node-1]. Reason: This node has insufficient available memory.") + ); + } + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId2); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-1")); + assertThat(assignment.getNodeRoutingTable().get("node-1").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + } + + public void testRebalance_GivenNormalPriorityModelToLoad_AndLowPriorityModelCanStay() throws Exception { + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 4); + DiscoveryNode node2 = buildNode("node-2", nodeMemoryBytes, 2); + + Map nodeLoads = new HashMap<>(); + nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + nodeLoads.put(node1, NodeLoad.builder("node-2").setMaxMemory(nodeMemoryBytes).build()); + + String modelId1 = "model-1"; + StartTrainedModelDeploymentAction.TaskParams taskParams1 = lowPriorityParams(modelId1, ByteSizeValue.ofMb(1).getBytes()); + String modelId2 = "model-2"; + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, ByteSizeValue.ofMb(1).getBytes(), 1, 4); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment( + modelId1, + TrainedModelAssignment.Builder.empty(taskParams1).addRoutingEntry("node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")) + ) + .build(); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(List.of(), List.of(node1, node2)), + Optional.of(taskParams2) + ).rebalance().build(); + + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId1); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTED)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-1")); + assertThat(assignment.getNodeRoutingTable().get("node-1").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getState(), equalTo(RoutingState.STARTED)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId2); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-1")); + assertThat(assignment.getNodeRoutingTable().get("node-1").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + } + + public void testRebalance_GivenNormalPriorityModelToLoad_AndLowPriorityModelMustRelocate() throws Exception { + long nodeMemoryBytes = ByteSizeValue.ofGb(1).getBytes(); + DiscoveryNode node1 = buildNode("node-1", nodeMemoryBytes, 4); + DiscoveryNode node2 = buildNode("node-2", nodeMemoryBytes, 2); + + Map nodeLoads = new HashMap<>(); + nodeLoads.put(node1, NodeLoad.builder("node-1").setMaxMemory(nodeMemoryBytes).build()); + nodeLoads.put(node2, NodeLoad.builder("node-2").setMaxMemory(nodeMemoryBytes).build()); + + String modelId1 = "model-1"; + StartTrainedModelDeploymentAction.TaskParams taskParams1 = lowPriorityParams(modelId1, ByteSizeValue.ofMb(300).getBytes()); + String modelId2 = "model-2"; + StartTrainedModelDeploymentAction.TaskParams taskParams2 = normalPriorityParams(modelId2, ByteSizeValue.ofMb(300).getBytes(), 1, 4); + TrainedModelAssignmentMetadata currentMetadata = TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment( + modelId1, + TrainedModelAssignment.Builder.empty(taskParams1).addRoutingEntry("node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")) + ) + .build(); + + TrainedModelAssignmentMetadata result = new TrainedModelAssignmentRebalancer( + currentMetadata, + nodeLoads, + Map.of(List.of(), List.of(node1, node2)), + Optional.of(taskParams2) + ).rebalance().build(); + + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId1); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-2")); + assertThat(assignment.getNodeRoutingTable().get("node-2").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-2").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-2").getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + { + TrainedModelAssignment assignment = result.getModelAssignment(modelId2); + assertThat(assignment, is(notNullValue())); + assertThat(assignment.getAssignmentState(), equalTo(AssignmentState.STARTING)); + assertThat(assignment.getNodeRoutingTable(), is(aMapWithSize(1))); + assertThat(assignment.getNodeRoutingTable(), hasKey("node-1")); + assertThat(assignment.getNodeRoutingTable().get("node-1").getCurrentAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getTargetAllocations(), equalTo(1)); + assertThat(assignment.getNodeRoutingTable().get("node-1").getState(), equalTo(RoutingState.STARTING)); + assertThat(assignment.getReason().isPresent(), is(false)); + } + } + + private static StartTrainedModelDeploymentAction.TaskParams lowPriorityParams(String modelId, long modelSize) { + return new StartTrainedModelDeploymentAction.TaskParams( + modelId, + modelSize, + 1, + 1, + 1024, + ByteSizeValue.ofBytes(modelSize), + Priority.LOW + ); + } + + private static StartTrainedModelDeploymentAction.TaskParams normalPriorityParams( String modelId, long modelSize, int numberOfAllocations, @@ -596,7 +1033,8 @@ private static StartTrainedModelDeploymentAction.TaskParams newParams( numberOfAllocations, threadsPerAllocation, 1024, - ByteSizeValue.ofBytes(modelSize) + ByteSizeValue.ofBytes(modelSize), + Priority.NORMAL ); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AllocationReducerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AllocationReducerTests.java index b81f69becab7a..90322c8a62833 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AllocationReducerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/planning/AllocationReducerTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; @@ -175,7 +176,8 @@ private static TrainedModelAssignment createAssignment( numberOfAllocations, randomIntBetween(1, 16), 1024, - null + null, + Priority.NORMAL ) ); allocationsByNode.entrySet() diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/planning/ZoneAwareAssignmentPlannerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/planning/ZoneAwareAssignmentPlannerTests.java index 58a1922147061..6d7ba9c2b3023 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/planning/ZoneAwareAssignmentPlannerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/planning/ZoneAwareAssignmentPlannerTests.java @@ -111,6 +111,9 @@ public void testGivenOneModel_OneNodePerZone_TwoZones_PartiallyFits() { Map> indexedBasedPlan = convertToIdIndexed(plan); assertThat(indexedBasedPlan.keySet(), hasItems("m_1")); assertThat(indexedBasedPlan.get("m_1"), equalTo(Map.of("n_1", 1, "n_2", 1))); + + assertThat(plan.getRemainingNodeMemory("n_1"), equalTo(0L)); + assertThat(plan.getRemainingNodeMemory("n_2"), equalTo(0L)); } public void testGivenThreeModels_TwoNodesPerZone_ThreeZones_FullyFit() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTaskTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTaskTests.java index 70cabafdb4501..e343ecc8dc29a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTaskTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTaskTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.PassThroughConfig; import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentNodeService; import org.mockito.ArgumentCaptor; @@ -56,7 +57,8 @@ void assertTrackingComplete(Consumer method, String randomInt(5), randomInt(5), randomInt(5), - randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, Long.MAX_VALUE)) + randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, Long.MAX_VALUE)), + Priority.NORMAL ), nodeService, licenseState, @@ -88,7 +90,8 @@ public void testUpdateNumberOfAllocations() { randomIntBetween(1, 32), randomIntBetween(1, 32), randomInt(5), - randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, Long.MAX_VALUE)) + randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, Long.MAX_VALUE)), + randomFrom(Priority.values()) ); TrainedModelDeploymentTask task = new TrainedModelDeploymentTask( @@ -113,5 +116,6 @@ public void testUpdateNumberOfAllocations() { assertThat(updatedParams.getNumberOfAllocations(), equalTo(newNumberOfAllocations)); assertThat(updatedParams.getThreadsPerAllocation(), equalTo(initialParams.getThreadsPerAllocation())); assertThat(updatedParams.getCacheSize(), equalTo(initialParams.getCacheSize())); + assertThat(updatedParams.getPriority(), equalTo(initialParams.getPriority())); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchBuilderTests.java index 355bacd6c743b..19d68e316c22a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchBuilderTests.java @@ -7,7 +7,10 @@ package org.elasticsearch.xpack.ml.inference.pytorch.process; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.TaskParams; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.ml.process.NativeController; import org.elasticsearch.xpack.ml.process.ProcessPipes; import org.junit.Before; @@ -44,7 +47,12 @@ public void setUpMocks() { } public void testBuild() throws IOException, InterruptedException { - new PyTorchBuilder(nativeController, processPipes, 2, 4, 12).build(); + + new PyTorchBuilder( + nativeController, + processPipes, + new TaskParams("my_model", 42L, 4, 2, 1024, ByteSizeValue.ofBytes(12), Priority.NORMAL) + ).build(); verify(nativeController).startProcess(commandCaptor.capture()); @@ -62,7 +70,8 @@ public void testBuild() throws IOException, InterruptedException { } public void testBuildWithNoCache() throws IOException, InterruptedException { - new PyTorchBuilder(nativeController, processPipes, 2, 4, 0).build(); + new PyTorchBuilder(nativeController, processPipes, new TaskParams("my_model", 42L, 4, 2, 1024, ByteSizeValue.ZERO, Priority.NORMAL)) + .build(); verify(nativeController).startProcess(commandCaptor.capture()); @@ -77,4 +86,27 @@ public void testBuildWithNoCache() throws IOException, InterruptedException { ) ); } + + public void testBuildWithLowPriority() throws IOException, InterruptedException { + new PyTorchBuilder( + nativeController, + processPipes, + new TaskParams("my_model", 42L, 1, 1, 1024, ByteSizeValue.ofBytes(42), Priority.LOW) + ).build(); + + verify(nativeController).startProcess(commandCaptor.capture()); + + assertThat( + commandCaptor.getValue(), + contains( + "./pytorch_inference", + "--validElasticLicenseKeyConfirmed=true", + "--numThreadsPerAllocation=1", + "--numAllocations=1", + "--cacheMemorylimitBytes=42", + "--lowPriority", + PROCESS_PIPES_ARG + ) + ); + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/NodeLoadDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/NodeLoadDetectorTests.java index 7b39f527db173..bb29f75dd566c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/NodeLoadDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/NodeLoadDetectorTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; @@ -132,7 +133,8 @@ public void testNodeLoadDetection() { 1, 1, 1024, - ByteSizeValue.ofBytes(MODEL_MEMORY_REQUIREMENT) + ByteSizeValue.ofBytes(MODEL_MEMORY_REQUIREMENT), + Priority.NORMAL ) ) .addRoutingEntry("_node_id4", new RoutingInfo(1, 1, RoutingState.STARTING, "")) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/3rd_party_deployment.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/3rd_party_deployment.yml index 0a84fd88c9448..8e99e3f0c31d7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/3rd_party_deployment.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/3rd_party_deployment.yml @@ -135,6 +135,39 @@ setup: model_id: test_model - match: { stopped: true } --- +"Test start and stop deployment with low priority": + - do: + ml.start_trained_model_deployment: + model_id: test_model + priority: "low" + wait_for: started + - match: { assignment.assignment_state: started } + - match: { assignment.task_parameters.model_id: test_model } + - match: { assignment.task_parameters.priority: low } + - match: { assignment.task_parameters.number_of_allocations: 1 } + - match: { assignment.task_parameters.threads_per_allocation: 1 } + + - do: + ml.stop_trained_model_deployment: + model_id: test_model + - match: { stopped: true } +--- +"Test start deployment with low priority and multiple allocations": + - do: + catch: /\[number_of_allocations\] must be 1 when \[priority\] is low/ + ml.start_trained_model_deployment: + model_id: test_model + priority: "low" + number_of_allocations: 3 +--- +"Test start deployment with low priority and multiple threads per allocation": + - do: + catch: /\[threads_per_allocation\] must be 1 when \[priority\] is low/ + ml.start_trained_model_deployment: + model_id: test_model + priority: "low" + threads_per_allocation: 4 +--- "Test update deployment": - do: ml.start_trained_model_deployment: From 3295662697f2d414dddeb8cae345f048124924e2 Mon Sep 17 00:00:00 2001 From: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> Date: Fri, 4 Nov 2022 09:18:35 -0400 Subject: [PATCH 07/17] [DOCS] Add time range info to TSDS docs (#91291) * [DOCS] Add time range info to TSDS docs * Fixup --- .../data-streams/set-up-tsds.asciidoc | 5 +++-- docs/reference/data-streams/tsds.asciidoc | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/reference/data-streams/set-up-tsds.asciidoc b/docs/reference/data-streams/set-up-tsds.asciidoc index 3c08dbc6107f4..bee0d40ece4a6 100644 --- a/docs/reference/data-streams/set-up-tsds.asciidoc +++ b/docs/reference/data-streams/set-up-tsds.asciidoc @@ -279,8 +279,9 @@ To automatically create your TSDS, submit an indexing request that targets the TSDS's name. This name must match one of your index template's index patterns. -To test the following example, update the timestamps to within the 24 hours after -your current time. +IMPORTANT: To test the following example, update the timestamps to within three hours of +your current time. Data added to a TSDS must always fall within an +<>. [source,console] ---- diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index 9a85d3d1edda7..a454e1652e399 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -242,6 +242,26 @@ value borders the `index.time_series.start_time` for the new write index. This ensures the `@timestamp` ranges for neighboring backing indices always border but never overlap. +[discrete] +[[tsds-accepted-time-range]] +==== Accepted time range for adding data + +A TSDS is designed to ingest current metrics data. When the TSDS is first +created the initial backing index has: + +* an `index.time_series.start_time` value set to `now - index.look_ahead_time` +* an `index.time_series.end_time` value set to `now + index.look_ahead_time` + +Only data that falls inside that range can be indexed. + +In our <>, +`index.look_ahead_time` is set to three hours, so only documents with a +`@timestamp` value that is within three hours previous or subsequent to the +present time are accepted for indexing. + +You can use the <> to check the +accepted time range for writing to any TSDS. + [discrete] [[dimension-based-routing]] ==== Dimension-based routing From 73ad49ac51d2bc9af405f976239730f84034981b Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Fri, 4 Nov 2022 14:20:44 +0100 Subject: [PATCH 08/17] Rename NamedComponent name parameter to value (#91306) to allow @NamedComponent("name") syntax java annotation should have single value annotation relates #88980 --- .../plugin/StablePluginBuildPluginFuncTest.groovy | 2 +- .../gradle/plugin/scanner/NamedComponentScanner.java | 4 ++-- .../plugin/scanner/NamedComponentScannerSpec.groovy | 8 ++++---- .../org/elasticsearch/plugin/api/NamedComponent.java | 2 +- .../scanner/test_classes/TestNamedComponent.java | 2 +- docs/changelog/91306.yaml | 5 +++++ .../java/org/elasticsearch/plugin/api/Nameable.java | 2 +- .../org/elasticsearch/plugin/api/NamedComponent.java | 2 +- .../example/analysis/ExampleAnalyzerFactory.java | 2 +- .../example/analysis/ExampleCharFilterFactory.java | 2 +- .../example/analysis/ExampleTokenFilterFactory.java | 2 +- .../example/analysis/ExampleTokenizerFactory.java | 2 +- .../indices/analysis/AnalysisModuleTests.java | 8 ++++---- .../analysis/wrappers/StableApiWrappersTests.java | 10 +++++----- .../org/elasticsearch/plugins/PluginsServiceTests.java | 2 +- 15 files changed, 30 insertions(+), 25 deletions(-) create mode 100644 docs/changelog/91306.yaml diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy index 2741764a28cf2..fc706497008a6 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy @@ -90,7 +90,7 @@ class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest { import org.elasticsearch.plugin.api.NamedComponent; import org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass; - @NamedComponent(name = "componentA") + @NamedComponent( "componentA") public class A extends ExtensibleClass { } """ diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/NamedComponentScanner.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/NamedComponentScanner.java index dc8c6c740ac14..86843c8f20eee 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/NamedComponentScanner.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/NamedComponentScanner.java @@ -31,8 +31,8 @@ public Map> scanForNamedClasses(Collection new AnnotationVisitor(Opcodes.ASM9) { @Override - public void visit(String name, Object value) { - assert name.equals("name"); + public void visit(String key, Object value) { + assert key.equals("value"); assert value instanceof String; map.put(value.toString(), classname); } diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/NamedComponentScannerSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/NamedComponentScannerSpec.groovy index f6ade83168cc2..fc48ae6a0ad67 100644 --- a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/NamedComponentScannerSpec.groovy +++ b/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/NamedComponentScannerSpec.groovy @@ -76,7 +76,7 @@ class NamedComponentScannerSpec extends Specification { package p; import org.elasticsearch.plugin.api.*; import org.elasticsearch.plugin.scanner.test_classes.*; - @NamedComponent(name = "a_component") + @NamedComponent("a_component") public class A extends ExtensibleClass {} """ ), "p/B.class", InMemoryJavaCompiler.compile( @@ -84,7 +84,7 @@ class NamedComponentScannerSpec extends Specification { package p; import org.elasticsearch.plugin.api.*; import org.elasticsearch.plugin.scanner.test_classes.*; - @NamedComponent(name = "b_component") + @NamedComponent("b_component") public class B implements ExtensibleInterface{} """ ) @@ -136,7 +136,7 @@ class NamedComponentScannerSpec extends Specification { package p; import org.elasticsearch.plugin.api.*; import org.elasticsearch.plugin.scanner.test_classes.*; - @NamedComponent(name = "a_component") + @NamedComponent("a_component") public class A extends CustomExtensibleClass {} """, "p.B", @@ -144,7 +144,7 @@ class NamedComponentScannerSpec extends Specification { package p; import org.elasticsearch.plugin.api.*; import org.elasticsearch.plugin.scanner.test_classes.*; - @NamedComponent(name = "b_component") + @NamedComponent("b_component") public class B implements CustomExtensibleInterface{} """ ); diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/NamedComponent.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/NamedComponent.java index 7edf5ef73493c..65a1a0d46abd7 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/NamedComponent.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/NamedComponent.java @@ -20,5 +20,5 @@ * The name used for registration and lookup * @return a name */ - String name(); + String value(); } diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java index e3527c94426f7..a98e6b08a7972 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java @@ -8,7 +8,7 @@ package org.elasticsearch.plugin.scanner.test_classes; -@org.elasticsearch.plugin.api.NamedComponent(name = "test_named_component") +@org.elasticsearch.plugin.api.NamedComponent("test_named_component") public class TestNamedComponent implements ExtensibleInterface { } diff --git a/docs/changelog/91306.yaml b/docs/changelog/91306.yaml new file mode 100644 index 0000000000000..4170ecd1973ac --- /dev/null +++ b/docs/changelog/91306.yaml @@ -0,0 +1,5 @@ +pr: 91306 +summary: Rename `NamedComponent` name parameter to value +area: Infra/Plugins +type: enhancement +issues: [] diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Nameable.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Nameable.java index c02757260c89c..79b9c5298ffce 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Nameable.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Nameable.java @@ -22,7 +22,7 @@ public interface Nameable { default String name() { NamedComponent[] annotationsByType = this.getClass().getAnnotationsByType(NamedComponent.class); if (annotationsByType.length == 1) { - return annotationsByType[0].name(); + return annotationsByType[0].value(); } return null; } diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/NamedComponent.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/NamedComponent.java index a8a0511e35cc2..ca62f8dd5aabd 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/NamedComponent.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/NamedComponent.java @@ -23,5 +23,5 @@ * The name used for registration and lookup * @return a name */ - String name(); + String value(); } diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalyzerFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalyzerFactory.java index 2fa395251f585..053b8fffb4512 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalyzerFactory.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalyzerFactory.java @@ -14,7 +14,7 @@ import org.elasticsearch.example.analysis.lucene.UnderscoreTokenizer; import org.elasticsearch.plugin.api.NamedComponent; -@NamedComponent(name = "example_analyzer_factory") +@NamedComponent( "example_analyzer_factory") public class ExampleAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory { @Override diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleCharFilterFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleCharFilterFactory.java index 3a10b0431ebc4..e918aa77b6cca 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleCharFilterFactory.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleCharFilterFactory.java @@ -15,7 +15,7 @@ import java.io.Reader; -@NamedComponent(name = "example_char_filter") +@NamedComponent( "example_char_filter") public class ExampleCharFilterFactory implements CharFilterFactory { @Override public Reader create(Reader reader) { diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenFilterFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenFilterFactory.java index 219b6bb5dd98f..4f8e20455c36f 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenFilterFactory.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenFilterFactory.java @@ -14,7 +14,7 @@ import org.elasticsearch.plugin.analysis.api.AnalysisMode; import org.elasticsearch.plugin.api.NamedComponent; -@NamedComponent(name = "example_token_filter_factory") +@NamedComponent( "example_token_filter_factory") public class ExampleTokenFilterFactory implements org.elasticsearch.plugin.analysis.api.TokenFilterFactory { @Override public TokenStream create(TokenStream tokenStream) { diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenizerFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenizerFactory.java index bfaaf94f6604c..dbd55fd9dfe55 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenizerFactory.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenizerFactory.java @@ -13,7 +13,7 @@ import org.elasticsearch.plugin.analysis.api.TokenizerFactory; import org.elasticsearch.plugin.api.NamedComponent; -@NamedComponent(name = "example_tokenizer_factory") +@NamedComponent( "example_tokenizer_factory") public class ExampleTokenizerFactory implements TokenizerFactory { @Override public Tokenizer create() { diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 74fa227c8051d..68913d53c4f01 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -476,7 +476,7 @@ public Map getHunspellDictionaries() { assertSame(dictionary, module.getHunspellService().getDictionary("foo")); } - @NamedComponent(name = "stableCharFilterFactory") + @NamedComponent("stableCharFilterFactory") public static class TestCharFilterFactory implements org.elasticsearch.plugin.analysis.api.CharFilterFactory { @SuppressForbidden(reason = "need a public constructor") public TestCharFilterFactory() {} @@ -506,7 +506,7 @@ private static NormalizeCharMap charMap() { } } - @NamedComponent(name = "stableTokenFilterFactory") + @NamedComponent("stableTokenFilterFactory") public static class TestTokenFilterFactory implements org.elasticsearch.plugin.analysis.api.TokenFilterFactory { @SuppressForbidden(reason = "need a public constructor") @@ -544,7 +544,7 @@ protected boolean accept() throws IOException { } } - @NamedComponent(name = "stableTokenizerFactory") + @NamedComponent("stableTokenizerFactory") public static class TestTokenizerFactory implements org.elasticsearch.plugin.analysis.api.TokenizerFactory { @SuppressForbidden(reason = "need a public constructor") public TestTokenizerFactory() {} @@ -564,7 +564,7 @@ protected boolean isTokenChar(int c) { } } - @NamedComponent(name = "stableAnalyzerFactory") + @NamedComponent("stableAnalyzerFactory") public static class TestAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory { @Override diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappersTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappersTests.java index 4f960a0e2e427..508d0c77f6f83 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappersTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappersTests.java @@ -197,7 +197,7 @@ public void testCharFilterFactoryDelegation() throws IOException { assertThat(charFilterFactory.name(), equalTo("TestCharFilterFactory")); } - @NamedComponent(name = "DefaultConstrAnalyzerFactory") + @NamedComponent("DefaultConstrAnalyzerFactory") public static class DefaultConstrAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory { public DefaultConstrAnalyzerFactory(int x) {} @@ -209,7 +209,7 @@ public Analyzer create() { } - @NamedComponent(name = "TestAnalyzerFactory") + @NamedComponent("TestAnalyzerFactory") public static class TestAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory { @Override @@ -219,7 +219,7 @@ public Analyzer create() { } - @NamedComponent(name = "TestTokenizerFactory") + @NamedComponent("TestTokenizerFactory") public static class TestTokenizerFactory implements org.elasticsearch.plugin.analysis.api.TokenizerFactory { @Override @@ -228,7 +228,7 @@ public Tokenizer create() { } } - @NamedComponent(name = "TestTokenFilterFactory") + @NamedComponent("TestTokenFilterFactory") public static class TestTokenFilterFactory implements org.elasticsearch.plugin.analysis.api.TokenFilterFactory { @Override @@ -253,7 +253,7 @@ public AnalysisMode getAnalysisMode() { } } - @NamedComponent(name = "TestCharFilterFactory") + @NamedComponent("TestCharFilterFactory") public static class TestCharFilterFactory implements org.elasticsearch.plugin.analysis.api.CharFilterFactory { @Override diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index 47b81da36b8e3..d06410ff0ac23 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -795,7 +795,7 @@ public void testStablePluginLoading() throws Exception { import org.elasticsearch.plugin.analysis.api.CharFilterFactory; import org.elasticsearch.plugin.api.NamedComponent; import java.io.Reader; - @NamedComponent(name = "a_name") + @NamedComponent( "a_name") public class A implements CharFilterFactory { @Override public Reader create(Reader reader) { From 163f218078d57c8c159bfde7b080f590113e78f4 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 4 Nov 2022 14:58:31 +0100 Subject: [PATCH 09/17] Fix TransportResponse reference counting in DirectResponseChannel (#91289) In #76474 we fixed a circuit breaker leak in TransportActionProxy by incrementing a reference on the TransportResponse that is later decremented by the OutboundHandler. This works well for all cases except when the request targets the node which is also the proxy node. In that case the reference is incremented but will never be decremented as the local execution (using TransportService#localNodeConnection and DirectResponseChannel) bypasses the OutboundHandler. This change fixes the ref counting by also decrementing the TransportResponse in DirectResponseChannel. This will also have the consequence to correctly decrement used bytes of the request circuit breaker when GetCcrRestoreFileChunkResponse are executed on a node that is also a proxy node. --- docs/changelog/91289.yaml | 5 + .../transport/TransportService.java | 60 +++++++----- .../transport/TransportActionProxyTests.java | 96 ++++++++++++++++++- 3 files changed, 132 insertions(+), 29 deletions(-) create mode 100644 docs/changelog/91289.yaml diff --git a/docs/changelog/91289.yaml b/docs/changelog/91289.yaml new file mode 100644 index 0000000000000..724c3b93f4795 --- /dev/null +++ b/docs/changelog/91289.yaml @@ -0,0 +1,5 @@ +pr: 91289 +summary: Fix `TransportActionProxy` for local execution +area: Network +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index e935bb1e1578e..df7b15f817c79 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -1411,33 +1411,43 @@ public String getProfileName() { @Override public void sendResponse(TransportResponse response) throws IOException { - service.onResponseSent(requestId, action, response); - try (var shutdownBlock = service.pendingDirectHandlers.withRef()) { - if (shutdownBlock == null) { - // already shutting down, the handler will be completed by sendRequestInternal or doStop - return; - } - final TransportResponseHandler handler = service.responseHandlers.onResponseReceived(requestId, service); - if (handler == null) { - // handler already completed, likely by a timeout which is logged elsewhere - return; - } - final String executor = handler.executor(); - if (ThreadPool.Names.SAME.equals(executor)) { - processResponse(handler, response); - } else { - threadPool.executor(executor).execute(new ForkingResponseHandlerRunnable(handler, null) { - @Override - protected void doRun() { - processResponse(handler, response); - } + try { + service.onResponseSent(requestId, action, response); + try (var shutdownBlock = service.pendingDirectHandlers.withRef()) { + if (shutdownBlock == null) { + // already shutting down, the handler will be completed by sendRequestInternal or doStop + return; + } + final TransportResponseHandler handler = service.responseHandlers.onResponseReceived(requestId, service); + if (handler == null) { + // handler already completed, likely by a timeout which is logged elsewhere + return; + } + final String executor = handler.executor(); + if (ThreadPool.Names.SAME.equals(executor)) { + processResponse(handler, response); + } else { + response.incRef(); + threadPool.executor(executor).execute(new ForkingResponseHandlerRunnable(handler, null) { + @Override + protected void doRun() { + processResponse(handler, response); + } - @Override - public String toString() { - return "delivery of response to [" + requestId + "][" + action + "]: " + response; - } - }); + @Override + public void onAfter() { + response.decRef(); + } + + @Override + public String toString() { + return "delivery of response to [" + requestId + "][" + action + "]: " + response; + } + }); + } } + } finally { + response.decRef(); } } diff --git a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java index ee5143498dd11..632e8cc27b0d6 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java @@ -14,7 +14,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -27,8 +29,10 @@ import java.io.IOException; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; public class TransportActionProxyTests extends ESTestCase { protected ThreadPool threadPool; @@ -76,8 +80,9 @@ private MockTransportService buildService(final Version version) { public void testSendMessage() throws InterruptedException { serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_A"); channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); }); final boolean cancellable = randomBoolean(); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); @@ -86,21 +91,24 @@ public void testSendMessage() throws InterruptedException { serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { assertThat(task instanceof CancellableTask, equalTo(cancellable)); assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_B"); + final SimpleTestResponse response = new SimpleTestResponse("TS_B"); channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); }); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { assertThat(task instanceof CancellableTask, equalTo(cancellable)); assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_C"); + final SimpleTestResponse response = new SimpleTestResponse("TS_C"); channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); }); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); - CountDownLatch latch = new CountDownLatch(1); + final CountDownLatch latch = new CountDownLatch(1); + // Node A -> Node B -> Node C serviceA.sendRequest( nodeB, TransportActionProxy.getProxyAction("internal:test"), @@ -133,6 +141,61 @@ public void handleException(TransportException exp) { latch.await(); } + public void testSendLocalRequest() throws InterruptedException { + final AtomicReference response = new AtomicReference<>(); + final boolean cancellable = randomBoolean(); + serviceB.registerRequestHandler( + "internal:test", + randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + SimpleTestRequest::new, + (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse responseB = new SimpleTestResponse("TS_B"); + channel.sendResponse(responseB); + response.set(responseB); + } + ); + TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); + AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); + + final CountDownLatch latch = new CountDownLatch(1); + // Node A -> Proxy Node B (Local execution) + serviceA.sendRequest( + nodeB, + TransportActionProxy.getProxyAction("internal:test"), + TransportActionProxy.wrapRequest(nodeB, new SimpleTestRequest("TS_A", cancellable)), // Request + new TransportResponseHandler() { + @Override + public SimpleTestResponse read(StreamInput in) throws IOException { + return new SimpleTestResponse(in); + } + + @Override + public void handleResponse(SimpleTestResponse response) { + try { + assertEquals("TS_B", response.targetNode); + } finally { + latch.countDown(); + } + } + + @Override + public void handleException(TransportException exp) { + try { + throw new AssertionError(exp); + } finally { + latch.countDown(); + } + } + } + ); + latch.await(); + + assertThat(response.get(), notNullValue()); + assertThat(response.get().hasReferences(), equalTo(false)); + } + public void testException() throws InterruptedException { boolean cancellable = randomBoolean(); serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { @@ -230,7 +293,12 @@ public boolean shouldCancelChildrenOnCancellation() { } public static class SimpleTestResponse extends TransportResponse { + final String targetNode; + final RefCounted refCounted = new AbstractRefCounted() { + @Override + protected void closeInternal() {} + }; SimpleTestResponse(String targetNode) { this.targetNode = targetNode; @@ -245,6 +313,26 @@ public static class SimpleTestResponse extends TransportResponse { public void writeTo(StreamOutput out) throws IOException { out.writeString(targetNode); } + + @Override + public void incRef() { + refCounted.incRef(); + } + + @Override + public boolean tryIncRef() { + return refCounted.tryIncRef(); + } + + @Override + public boolean decRef() { + return refCounted.decRef(); + } + + @Override + public boolean hasReferences() { + return refCounted.hasReferences(); + } } public void testGetAction() { From f2237dccd0a560c9f29489ff70e8e9519a43742d Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 4 Nov 2022 10:26:36 -0400 Subject: [PATCH 10/17] [ML] add ability to filter and sort buckets by change_point numeric values (#91299) The `change_point` aggregation automatically detects the most statistically impactful changepoint in a series of buckets. There is occasion to detect change points over many different timeseries facets. Some of those facets may have no changes at all, so, it would be prudent to filter those from the response. This commit allows the `p_value`, `type`, and internal bucket values to be used by other pipeline aggregations. Two specific and useful cases are: bucket_selector and bucket_sort. Allowing facets to be sorted by which had the largest changes and simply filtering out those with no changes at all. closes: https://github.com/elastic/elasticsearch/issues/91281 --- docs/changelog/91299.yaml | 5 + .../InternalChangePointAggregation.java | 14 + .../test/ml/change_point_agg.yml | 242 +++++++++++++++--- 3 files changed, 232 insertions(+), 29 deletions(-) create mode 100644 docs/changelog/91299.yaml diff --git a/docs/changelog/91299.yaml b/docs/changelog/91299.yaml new file mode 100644 index 0000000000000..9d50b2244b4a7 --- /dev/null +++ b/docs/changelog/91299.yaml @@ -0,0 +1,5 @@ +pr: 91299 +summary: Add ability to filter and sort buckets by `change_point` numeric values +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/InternalChangePointAggregation.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/InternalChangePointAggregation.java index 554083d87b346..e9713497cb148 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/InternalChangePointAggregation.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/InternalChangePointAggregation.java @@ -75,6 +75,20 @@ protected boolean mustReduceOnSingleInternalAgg() { @Override public Object getProperty(List path) { + if (path.size() == 1) { + String property = path.get(0); + if (property.equals("p_value")) { + return changeType.pValue(); + } + if (property.equals("type")) { + return changeType.getName(); + } + if (property.equals("change_point")) { + return changeType.changePoint(); + } + } else if (path.size() > 1 && path.get(0).equals("bucket") && bucket != null) { + return bucket.getProperty(name, path.subList(1, path.size())); + } return null; } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/change_point_agg.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/change_point_agg.yml index 678a0bcee2dec..3dc4fb54dbc13 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/change_point_agg.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/change_point_agg.yml @@ -13,6 +13,8 @@ setup: type: integer time: type: date + kind: + type: keyword - do: headers: @@ -36,53 +38,101 @@ setup: refresh: true body: | {"index":{}} - {"cost":200,"time":1587501233000} + {"cost":200,"time":1587501233000,"kind":"changed"} {"index":{}} - {"cost":200,"time":1587501243000} + {"cost":200,"time":1587501243000,"kind":"changed"} {"index":{}} - {"cost":200,"time":1587501253000} + {"cost":200,"time":1587501253000,"kind":"changed"} {"index":{}} - {"cost":250,"time":1587501263000} + {"cost":250,"time":1587501263000,"kind":"changed"} {"index":{}} - {"cost":250,"time":1587501273000} + {"cost":250,"time":1587501273000,"kind":"changed"} {"index":{}} - {"cost":580,"time":1587501283000} + {"cost":580,"time":1587501283000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501293000} + {"cost":600,"time":1587501293000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501303000} + {"cost":600,"time":1587501303000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501313000} + {"cost":600,"time":1587501313000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501313000} + {"cost":600,"time":1587501313000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501323000} + {"cost":600,"time":1587501323000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501333000} + {"cost":600,"time":1587501333000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501343000} + {"cost":600,"time":1587501343000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501353000} + {"cost":600,"time":1587501353000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501363000} + {"cost":600,"time":1587501363000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501373000} + {"cost":600,"time":1587501373000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501383000} + {"cost":600,"time":1587501383000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501393000} + {"cost":600,"time":1587501393000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501403000} + {"cost":600,"time":1587501403000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501413000} + {"cost":600,"time":1587501413000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501423000} + {"cost":600,"time":1587501423000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501433000} + {"cost":600,"time":1587501433000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501443000} + {"cost":600,"time":1587501443000,"kind":"changed"} {"index":{}} - {"cost":600,"time":1587501453000} + {"cost":600,"time":1587501453000,"kind":"changed"} + {"index":{}} + {"cost":600,"time":1587501233000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501243000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501253000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501263000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501273000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501283000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501293000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501303000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501313000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501313000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501323000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501333000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501343000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501353000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501363000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501373000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501383000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501393000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501403000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501413000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501423000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501433000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501443000,"kind":"unchanged"} + {"index":{}} + {"cost":600,"time":1587501453000,"kind":"unchanged"} --- "Test change_point agg simple": @@ -100,8 +150,8 @@ setup: "fixed_interval": "1s" }, "aggs": { - "avg": { - "avg": { + "min": { + "min": { "field": "cost" } } @@ -109,7 +159,7 @@ setup: }, "change_point": { "change_point": { - "buckets_path": "date>avg" + "buckets_path": "date>min" } } } @@ -196,8 +246,8 @@ setup: "fixed_interval": "1s" }, "aggs": { - "avg": { - "avg": { + "min": { + "min": { "field": "cost" } } @@ -211,3 +261,137 @@ setup: } } - is_true: aggregations.change_point.type.indeterminable +--- +"Test change_point with terms": + - do: + search: + index: store + size: 0 + body: > + { + "aggs": { + "terms": { + "terms": { + "field": "kind" + }, + "aggs": { + "date": { + "date_histogram": { + "field": "time", + "fixed_interval": "1s" + }, + "aggs": { + "min": { + "min": { + "field": "cost" + } + } + } + }, + "change_point": { + "change_point": { + "buckets_path": "date>min" + } + } + } + } + } + } + - is_true: aggregations.terms.buckets.0.change_point.type.trend_change + - is_true: aggregations.terms.buckets.0.change_point.type.trend_change.p_value + - is_true: aggregations.terms.buckets.0.change_point.type.trend_change.r_value + - is_false: aggregations.terms.buckets.1.change_point.type.trend_change + - is_false: aggregations.terms.buckets.1.change_point.type.trend_change.p_value + - is_false: aggregations.terms.buckets.1.change_point.type.trend_change.r_value + + - do: + search: + index: store + size: 0 + body: > + { + "aggs": { + "terms": { + "terms": { + "field": "kind" + }, + "aggs": { + "date": { + "date_histogram": { + "field": "time", + "fixed_interval": "1s" + }, + "aggs": { + "min": { + "min": { + "field": "cost" + } + } + } + }, + "change_point": { + "change_point": { + "buckets_path": "date>min" + } + }, + "select": { + "bucket_selector": { + "buckets_path": {"p_value": "change_point.p_value"}, + "script": "params.p_value < 0.5" + } + } + } + } + } + } + - length: {aggregations.terms.buckets: 1} + - is_true: aggregations.terms.buckets.0.change_point.type.trend_change + - is_true: aggregations.terms.buckets.0.change_point.type.trend_change.p_value + - is_true: aggregations.terms.buckets.0.change_point.type.trend_change.r_value + + - do: + search: + index: store + size: 0 + body: > + { + "aggs": { + "terms": { + "terms": { + "field": "kind" + }, + "aggs": { + "date": { + "date_histogram": { + "field": "time", + "fixed_interval": "1s" + }, + "aggs": { + "min": { + "min": { + "field": "cost" + } + } + } + }, + "change_point": { + "change_point": { + "buckets_path": "date>min" + } + }, + "sort": { + "bucket_sort": { + "sort": [{"change_point.p_value": {"order": "desc"}}] + } + } + } + } + } + } + - length: {aggregations.terms.buckets: 2} + - is_false: aggregations.terms.buckets.0.change_point.type.trend_change + - is_false: aggregations.terms.buckets.0.change_point.type.trend_change.p_value + - is_false: aggregations.terms.buckets.0.change_point.type.trend_change.r_value + - is_true: aggregations.terms.buckets.1.change_point.type.trend_change + - is_true: aggregations.terms.buckets.1.change_point.type.trend_change.p_value + - is_true: aggregations.terms.buckets.1.change_point.type.trend_change.r_value From d6cff11699a83fd0f6a43cc062abd6c28bbd8958 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 4 Nov 2022 16:10:55 +0100 Subject: [PATCH 11/17] Fix some warnings about scaled_float mappings in x-pack tests (#91304) Fixed some of these in the past, but these still remained. Wherever we load the monitoring templates we need the scaled float mapper available or the test logs get spammed with endless mapper parsing exceptions. --- .../xpack/ml/integration/AutodetectResultProcessorIT.java | 4 +++- x-pack/plugin/monitoring/build.gradle | 1 + .../xpack/monitoring/integration/MonitoringIT.java | 3 ++- .../xpack/monitoring/test/MonitoringIntegTestCase.java | 4 +++- .../elasticsearch/integration/DocumentLevelSecurityTests.java | 4 +++- .../elasticsearch/integration/FieldLevelSecurityTests.java | 4 +++- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java index f31f8df53247b..2334ce12cc991 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.ingest.common.IngestCommonPlugin; import org.elasticsearch.plugins.Plugin; @@ -133,7 +134,8 @@ protected Collection> getPlugins() { ReindexPlugin.class, MockPainlessScriptEngine.TestPlugin.class, // ILM is required for .ml-state template index settings - IndexLifecycle.class + IndexLifecycle.class, + MapperExtrasPlugin.class ); } diff --git a/x-pack/plugin/monitoring/build.gradle b/x-pack/plugin/monitoring/build.gradle index 7ec0e36c8c347..38f27012e3d6b 100644 --- a/x-pack/plugin/monitoring/build.gradle +++ b/x-pack/plugin/monitoring/build.gradle @@ -23,6 +23,7 @@ dependencies { testImplementation project(xpackModule('ilm')) testImplementation project(':modules:data-streams') testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(':modules:mapper-extras') } tasks.named("dependencyLicenses").configure { diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index 47a9e54644901..0b51e72846531 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; @@ -91,7 +92,7 @@ protected Settings nodeSettings() { @Override protected Collection> getPlugins() { - return Arrays.asList(LocalStateMonitoring.class, MockIngestPlugin.class, CommonAnalysisPlugin.class); + return Arrays.asList(LocalStateMonitoring.class, MockIngestPlugin.class, CommonAnalysisPlugin.class, MapperExtrasPlugin.class); } private String createBulkEntity() { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java index 8a7147fe301e0..43b4d83a5a301 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.store.MockFSIndexStore; @@ -73,7 +74,8 @@ protected Collection> nodePlugins() { LocalStateMonitoring.class, MockClusterAlertScriptEngine.TestPlugin.class, MockIngestPlugin.class, - CommonAnalysisPlugin.class + CommonAnalysisPlugin.class, + MapperExtrasPlugin.class ); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 7602981455b51..ad586e3e83f0a 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.index.query.FuzzyQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -116,7 +117,8 @@ protected Collection> nodePlugins() { ParentJoinPlugin.class, InternalSettingsPlugin.class, SpatialPlugin.class, - PercolatorPlugin.class + PercolatorPlugin.class, + MapperExtrasPlugin.class ); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index 0feb8f4b03add..ca704f42fa7f5 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.IndicesRequestCache; @@ -100,7 +101,8 @@ protected Collection> nodePlugins() { ParentJoinPlugin.class, InternalSettingsPlugin.class, PercolatorPlugin.class, - SpatialPlugin.class + SpatialPlugin.class, + MapperExtrasPlugin.class ); } From 72f3578f2b438ec61949fbdc8a3aab0f59aa3753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Fri, 4 Nov 2022 16:13:55 +0100 Subject: [PATCH 12/17] Store write load in IndexMetadata during data streams rollovers (#91019) This commits stores the index write load of the current data stream write-index during rollover into its IndexMetadata. Closes #91046 --- docs/changelog/91019.yaml | 5 + .../datastreams/DataStreamIT.java | 136 +++++++++++- .../DataStreamGetWriteIndexTests.java | 2 +- ...etadataDataStreamRolloverServiceTests.java | 16 +- .../rollover/MetadataRolloverService.java | 17 +- .../rollover/TransportRolloverAction.java | 19 +- .../cluster/metadata/IndexMetadata.java | 78 ++++++- .../index/shard/IndexWriteLoad.java | 200 ++++++++++++++++++ .../index/shard/IndexingStats.java | 34 ++- .../index/shard/InternalIndexingStats.java | 5 +- .../cluster/node/stats/NodeStatsTests.java | 1 + .../MetadataRolloverServiceTests.java | 12 +- .../cluster/ClusterStateTests.java | 22 ++ .../cluster/metadata/IndexMetadataTests.java | 14 ++ .../IndexWriteLoadSerializationTests.java | 60 ++++++ .../index/shard/IndexWriteLoadTests.java | 147 +++++++++++++ .../indices/IndexStatsMonitoringDocTests.java | 2 +- .../IndicesStatsMonitoringDocTests.java | 2 +- .../node/NodeStatsMonitoringDocTests.java | 15 +- 19 files changed, 749 insertions(+), 38 deletions(-) create mode 100644 docs/changelog/91019.yaml create mode 100644 server/src/main/java/org/elasticsearch/index/shard/IndexWriteLoad.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadSerializationTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadTests.java diff --git a/docs/changelog/91019.yaml b/docs/changelog/91019.yaml new file mode 100644 index 0000000000000..5753ab4d7fba3 --- /dev/null +++ b/docs/changelog/91019.yaml @@ -0,0 +1,5 @@ +pr: 91019 +summary: Store write load in the `IndexMetadata` during data streams rollovers +area: Allocation +type: enhancement +issues: [] diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 1fae35a7bf1b4..f3b1f0d2ac582 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -27,7 +27,11 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; @@ -61,6 +65,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -72,6 +78,8 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.shard.IndexWriteLoad; +import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.plugins.Plugin; @@ -80,6 +88,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; import org.elasticsearch.xcontent.XContentType; @@ -115,6 +125,8 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasItemInArray; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -127,7 +139,7 @@ public class DataStreamIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return List.of(DataStreamsPlugin.class); + return List.of(DataStreamsPlugin.class, MockTransportService.TestPlugin.class); } public void testBasicScenario() throws Exception { @@ -1998,6 +2010,128 @@ public void testSearchWithRouting() throws IOException, ExecutionException, Inte assertEquals(searchResponse.getTotalShards(), 4); } + public void testWriteIndexWriteLoadIsStoredAfterRollover() throws Exception { + final String dataStreamName = "logs-es"; + final int numberOfShards = randomIntBetween(1, 5); + final int numberOfReplicas = randomIntBetween(0, 1); + final var indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) + .build(); + DataStreamIT.putComposableIndexTemplate("my-template", null, List.of("logs-*"), indexSettings, null); + final var request = new CreateDataStreamAction.Request(dataStreamName); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet()); + + assertBusy(() -> { + for (int i = 0; i < 10; i++) { + indexDocs(dataStreamName, randomIntBetween(100, 200)); + } + + final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state(); + final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + final String writeIndex = dataStream.getWriteIndex().getName(); + final IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(writeIndex).get(); + for (IndexShardStats indexShardStats : indicesStatsResponse.getIndex(writeIndex).getIndexShards().values()) { + for (ShardStats shard : indexShardStats.getShards()) { + final IndexingStats.Stats shardIndexingStats = shard.getStats().getIndexing().getTotal(); + // Ensure that we have enough clock granularity before rolling over to ensure that we capture _some_ write load + assertThat(shardIndexingStats.getTotalActiveTimeInMillis(), is(greaterThan(0L))); + assertThat(shardIndexingStats.getWriteLoad(), is(greaterThan(0.0))); + } + } + }); + + assertAcked(client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet()); + final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state(); + final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + + for (Index index : dataStream.getIndices()) { + final IndexMetadata indexMetadata = clusterState.metadata().index(index); + final IndexWriteLoad indexWriteLoad = indexMetadata.getWriteLoad(); + + if (index.equals(dataStream.getWriteIndex()) == false) { + assertThat(indexWriteLoad, is(notNullValue())); + for (int shardId = 0; shardId < numberOfShards; shardId++) { + assertThat(indexWriteLoad.getWriteLoadForShard(shardId).getAsDouble(), is(greaterThanOrEqualTo(0.0))); + assertThat(indexWriteLoad.getUptimeInMillisForShard(shardId).getAsLong(), is(greaterThan(0L))); + } + } else { + assertThat(indexWriteLoad, is(nullValue())); + } + } + } + + public void testWriteLoadIsStoredInABestEffort() throws Exception { + // This test simulates the scenario where some nodes fail to respond + // to the IndicesStatsRequest and therefore only a partial view of the + // write-index write-load is stored during rollover. + // In this test we simulate the following scenario: + // - The DataStream template is configured to have 2 shards and 1 replica + // - There's an allocation rule to allocate the data stream nodes in 4 particular nodes + // - We want to simulate two possible cases here: + // - All the assigned nodes for shard 0 will fail to respond to the IndicesStatsRequest + // - Only the shard 1 replica will respond successfully to the IndicesStatsRequest ensuring that we fall back in that case + + final List dataOnlyNodes = internalCluster().startDataOnlyNodes(4); + final String dataStreamName = "logs-es"; + + final var indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes)) + .build(); + DataStreamIT.putComposableIndexTemplate("my-template", null, List.of("logs-*"), indexSettings, null); + final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + + for (int i = 0; i < 10; i++) { + indexDocs(dataStreamName, randomIntBetween(100, 200)); + } + + final ClusterState clusterStateBeforeRollover = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state(); + final DataStream dataStreamBeforeRollover = clusterStateBeforeRollover.getMetadata().dataStreams().get(dataStreamName); + final IndexRoutingTable currentDataStreamWriteIndexRoutingTable = clusterStateBeforeRollover.routingTable() + .index(dataStreamBeforeRollover.getWriteIndex()); + + final List failingIndicesStatsNodeIds = new ArrayList<>(); + for (ShardRouting shardRouting : currentDataStreamWriteIndexRoutingTable.shard(0).assignedShards()) { + failingIndicesStatsNodeIds.add(shardRouting.currentNodeId()); + } + failingIndicesStatsNodeIds.add(currentDataStreamWriteIndexRoutingTable.shard(1).primaryShard().currentNodeId()); + + for (String nodeId : failingIndicesStatsNodeIds) { + String nodeName = clusterStateBeforeRollover.nodes().resolveNode(nodeId).getName(); + MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeName); + transportService.addRequestHandlingBehavior( + IndicesStatsAction.NAME + "[n]", + (handler, request, channel, task) -> channel.sendResponse(new RuntimeException("Unable to get stats")) + ); + } + assertThat(failingIndicesStatsNodeIds.size(), is(equalTo(3))); + + assertAcked(client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet()); + final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state(); + final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + + for (Index index : dataStream.getIndices()) { + final IndexMetadata indexMetadata = clusterState.metadata().index(index); + final IndexWriteLoad indexWriteLoad = indexMetadata.getWriteLoad(); + + if (index.equals(dataStream.getWriteIndex()) == false) { + assertThat(indexWriteLoad, is(notNullValue())); + // All stats request performed against nodes holding the shard 0 failed + assertThat(indexWriteLoad.getWriteLoadForShard(0).isPresent(), is(false)); + assertThat(indexWriteLoad.getUptimeInMillisForShard(0).isPresent(), is(false)); + + // At least one of the shard 1 copies responded with stats + assertThat(indexWriteLoad.getWriteLoadForShard(1).getAsDouble(), is(greaterThanOrEqualTo(0.0))); + assertThat(indexWriteLoad.getUptimeInMillisForShard(1).getAsLong(), is(greaterThan(0L))); + } else { + assertThat(indexWriteLoad, is(nullValue())); + } + } + } + static void putComposableIndexTemplate( String id, @Nullable String mappings, diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index bbc04825a1d09..2b7cf6374e14e 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -313,7 +313,7 @@ private MetadataRolloverService.RolloverResult rolloverOver(ClusterState state, MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); List> metConditions = Collections.singletonList(condition); CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); - return rolloverService.rolloverClusterState(state, name, null, createIndexRequest, metConditions, time, false, false); + return rolloverService.rolloverClusterState(state, name, null, createIndexRequest, metConditions, time, false, false, null); } private Index getWriteIndex(ClusterState state, String name, String timestamp) { diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java index 67c763a31b0b7..f7d03528e826b 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -47,6 +48,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class MetadataDataStreamRolloverServiceTests extends ESTestCase { @@ -100,6 +102,7 @@ public void testRolloverClusterStateForDataStream() throws Exception { MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); List> metConditions = Collections.singletonList(condition); CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); + IndexWriteLoad indexWriteLoad = IndexWriteLoad.builder(1).build(); long before = testThreadPool.absoluteTimeInMillis(); MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState( @@ -110,7 +113,8 @@ public void testRolloverClusterStateForDataStream() throws Exception { metConditions, now, randomBoolean(), - false + false, + indexWriteLoad ); long after = testThreadPool.absoluteTimeInMillis(); @@ -138,12 +142,16 @@ public void testRolloverClusterStateForDataStream() throws Exception { IndexMetadata im = rolloverMetadata.index(rolloverMetadata.dataStreams().get(dataStreamName).getIndices().get(0)); Instant startTime1 = IndexSettings.TIME_SERIES_START_TIME.get(im.getSettings()); Instant endTime1 = IndexSettings.TIME_SERIES_END_TIME.get(im.getSettings()); + IndexWriteLoad indexWriteLoad1 = im.getWriteLoad(); im = rolloverMetadata.index(rolloverMetadata.dataStreams().get(dataStreamName).getIndices().get(1)); Instant startTime2 = IndexSettings.TIME_SERIES_START_TIME.get(im.getSettings()); Instant endTime2 = IndexSettings.TIME_SERIES_END_TIME.get(im.getSettings()); + IndexWriteLoad indexWriteLoad2 = im.getWriteLoad(); assertThat(startTime1.isBefore(endTime1), is(true)); assertThat(endTime1, equalTo(startTime2)); assertThat(endTime2.isAfter(endTime1), is(true)); + assertThat(indexWriteLoad1, is(equalTo(indexWriteLoad))); + assertThat(indexWriteLoad2, is(nullValue())); } finally { testThreadPool.shutdown(); } @@ -204,7 +212,8 @@ public void testRolloverAndMigrateDataStream() throws Exception { metConditions, now, randomBoolean(), - false + false, + null ); String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration()); @@ -295,7 +304,8 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting metConditions, now, randomBoolean(), - false + false, + null ); String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 9629b87b98ad6..447b5da508635 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -30,6 +30,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.snapshots.SnapshotInProgressException; @@ -98,7 +99,8 @@ public RolloverResult rolloverClusterState( List> metConditions, Instant now, boolean silent, - boolean onlyValidate + boolean onlyValidate, + @Nullable IndexWriteLoad sourceIndexWriteLoad ) throws Exception { validate(currentState.metadata(), rolloverTarget, newIndexName, createIndexRequest); final IndexAbstraction indexAbstraction = currentState.metadata().getIndicesLookup().get(rolloverTarget); @@ -121,7 +123,8 @@ public RolloverResult rolloverClusterState( metConditions, now, silent, - onlyValidate + onlyValidate, + sourceIndexWriteLoad ); default -> // the validate method above prevents this case @@ -228,7 +231,8 @@ private RolloverResult rolloverDataStream( List> metConditions, Instant now, boolean silent, - boolean onlyValidate + boolean onlyValidate, + @Nullable IndexWriteLoad sourceIndexWriteLoad ) throws Exception { if (SnapshotsService.snapshottingDataStreams(currentState, Collections.singleton(dataStream.getName())).isEmpty() == false) { @@ -284,10 +288,15 @@ private RolloverResult rolloverDataStream( ); RolloverInfo rolloverInfo = new RolloverInfo(dataStreamName, metConditions, threadPool.absoluteTimeInMillis()); + newState = ClusterState.builder(newState) .metadata( Metadata.builder(newState.metadata()) - .put(IndexMetadata.builder(newState.metadata().index(originalWriteIndex)).putRolloverInfo(rolloverInfo)) + .put( + IndexMetadata.builder(newState.metadata().index(originalWriteIndex)) + .indexWriteLoad(sourceIndexWriteLoad) + .putRolloverInfo(rolloverInfo) + ) ) .build(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index a5f9ed4a7384a..d10d607e0acaf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; @@ -38,6 +39,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -119,7 +121,8 @@ protected void masterOperation( IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getRolloverTarget()) .clear() .indicesOptions(IndicesOptions.fromOptions(true, false, true, true)) - .docs(true); + .docs(true) + .indexing(true); statsRequest.setParentTask(clusterService.localNode().getId(), task.getId()); // Rollover can sometimes happen concurrently, to handle these cases, we treat rollover in the same way we would treat a // "synchronized" block, in that we have a "before" world, where we calculate naming and condition matching, we then enter our @@ -292,9 +295,10 @@ public ClusterState executeTask( ); // Re-evaluate the conditions, now with our final source index name + IndexMetadata rolloverSourceIndex = currentState.metadata().index(rolloverNames.sourceName()); final Map postConditionResults = evaluateConditions( rolloverRequest.getConditions().values(), - buildStats(currentState.metadata().index(rolloverNames.sourceName()), rolloverTask.statsResponse()) + buildStats(rolloverSourceIndex, rolloverTask.statsResponse()) ); if (rolloverRequest.areConditionsMet(postConditionResults)) { @@ -304,6 +308,14 @@ public ClusterState executeTask( .filter(condition -> postConditionResults.get(condition.toString())) .toList(); + final IndexAbstraction rolloverTargetAbstraction = currentState.metadata() + .getIndicesLookup() + .get(rolloverRequest.getRolloverTarget()); + + final IndexWriteLoad sourceIndexWriteLoad = rolloverTargetAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM + ? IndexWriteLoad.fromStats(rolloverSourceIndex, rolloverTask.statsResponse()) + : null; + // Perform the actual rollover final var rolloverResult = rolloverService.rolloverClusterState( currentState, @@ -313,7 +325,8 @@ public ClusterState executeTask( metConditions, Instant.now(), false, - false + false, + sourceIndexWriteLoad ); results.add(rolloverResult); logger.trace("rollover result [{}]", rolloverResult); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 0ddf7592727c5..7cb1648ffbc62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexLongFieldRange; +import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.rest.RestStatus; @@ -521,11 +522,14 @@ public Iterator> settings() { static final String KEY_SYSTEM = "system"; static final String KEY_TIMESTAMP_RANGE = "timestamp_range"; public static final String KEY_PRIMARY_TERMS = "primary_terms"; + public static final String KEY_WRITE_LOAD = "write_load"; public static final String INDEX_STATE_FILE_PREFIX = "state-"; static final Version SYSTEM_INDEX_FLAG_ADDED = Version.V_7_10_0; + static final Version WRITE_LOAD_ADDED = Version.V_8_6_0; + private final int routingNumShards; private final int routingFactor; private final int routingPartitionSize; @@ -603,6 +607,8 @@ public Iterator> settings() { private final Instant timeSeriesStart; @Nullable private final Instant timeSeriesEnd; + @Nullable + private final IndexWriteLoad writeLoad; private IndexMetadata( final Index index, @@ -645,7 +651,8 @@ private IndexMetadata( @Nullable final IndexMode indexMode, @Nullable final Instant timeSeriesStart, @Nullable final Instant timeSeriesEnd, - final Version indexCompatibilityVersion + final Version indexCompatibilityVersion, + @Nullable final IndexWriteLoad writeLoad ) { this.index = index; this.version = version; @@ -696,6 +703,7 @@ private IndexMetadata( this.indexMode = indexMode; this.timeSeriesStart = timeSeriesStart; this.timeSeriesEnd = timeSeriesEnd; + this.writeLoad = writeLoad; assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards; } @@ -744,7 +752,8 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { this.indexMode, this.timeSeriesStart, this.timeSeriesEnd, - this.indexCompatibilityVersion + this.indexCompatibilityVersion, + this.writeLoad ); } @@ -799,7 +808,8 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) this.indexMode, this.timeSeriesStart, this.timeSeriesEnd, - this.indexCompatibilityVersion + this.indexCompatibilityVersion, + this.writeLoad ); } @@ -852,7 +862,8 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) { this.indexMode, this.timeSeriesStart, this.timeSeriesEnd, - this.indexCompatibilityVersion + this.indexCompatibilityVersion, + this.writeLoad ); } @@ -905,7 +916,8 @@ public IndexMetadata withTimestampRange(IndexLongFieldRange timestampRange) { this.indexMode, this.timeSeriesStart, this.timeSeriesEnd, - this.indexCompatibilityVersion + this.indexCompatibilityVersion, + this.writeLoad ); } @@ -954,7 +966,8 @@ public IndexMetadata withIncrementedVersion() { this.indexMode, this.timeSeriesStart, this.timeSeriesEnd, - this.indexCompatibilityVersion + this.indexCompatibilityVersion, + this.writeLoad ); } @@ -1145,6 +1158,11 @@ public MappingMetadata mapping() { return mapping; } + @Nullable + public IndexWriteLoad getWriteLoad() { + return writeLoad; + } + public static final String INDEX_RESIZE_SOURCE_UUID_KEY = "index.resize.source.uuid"; public static final String INDEX_RESIZE_SOURCE_NAME_KEY = "index.resize.source.name"; public static final Setting INDEX_RESIZE_SOURCE_UUID = Setting.simpleString(INDEX_RESIZE_SOURCE_UUID_KEY); @@ -1379,6 +1397,7 @@ private static class IndexMetadataDiff implements Diff { private final Diff> rolloverInfos; private final boolean isSystem; private final IndexLongFieldRange timestampRange; + private final IndexWriteLoad indexWriteLoad; IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); @@ -1412,6 +1431,7 @@ private static class IndexMetadataDiff implements Diff { rolloverInfos = DiffableUtils.diff(before.rolloverInfos, after.rolloverInfos, DiffableUtils.getStringKeySerializer()); isSystem = after.isSystem; timestampRange = after.timestampRange; + indexWriteLoad = after.writeLoad; } private static final DiffableUtils.DiffableValueReader ALIAS_METADATA_DIFF_VALUE_READER = @@ -1462,6 +1482,11 @@ private static class IndexMetadataDiff implements Diff { isSystem = false; } timestampRange = IndexLongFieldRange.readFrom(in); + if (in.getVersion().onOrAfter(WRITE_LOAD_ADDED)) { + indexWriteLoad = in.readOptionalWriteable(IndexWriteLoad::new); + } else { + indexWriteLoad = null; + } } @Override @@ -1492,6 +1517,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isSystem); } timestampRange.writeTo(out); + if (out.getVersion().onOrAfter(WRITE_LOAD_ADDED)) { + out.writeOptionalWriteable(indexWriteLoad); + } } @Override @@ -1518,6 +1546,7 @@ public IndexMetadata apply(IndexMetadata part) { builder.rolloverInfos.putAllFromMap(rolloverInfos.apply(part.rolloverInfos)); builder.system(isSystem); builder.timestampRange(timestampRange); + builder.indexWriteLoad(indexWriteLoad); return builder.build(); } } @@ -1579,6 +1608,10 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function PARSER = new ConstructingObjectParser<>( + "index_write_load_parser", + false, + (args, unused) -> IndexWriteLoad.create((List) args[0], (List) args[1]) + ); + + static { + PARSER.declareDoubleArray(ConstructingObjectParser.constructorArg(), SHARDS_WRITE_LOAD_FIELD); + PARSER.declareLongArray(ConstructingObjectParser.constructorArg(), SHARDS_UPTIME_IN_MILLIS); + } + + public static IndexWriteLoad create(List shardsWriteLoad, List shardsUptimeInMillis) { + if (shardsWriteLoad.size() != shardsUptimeInMillis.size()) { + assert false; + throw new IllegalArgumentException( + "The same number of shard write loads and shard uptimes should be provided, but " + + shardsWriteLoad + + " " + + shardsUptimeInMillis + + " were provided" + ); + } + + if (shardsWriteLoad.isEmpty()) { + assert false; + throw new IllegalArgumentException("At least one shard write load and uptime should be provided, but none was provided"); + } + + return new IndexWriteLoad( + shardsWriteLoad.stream().mapToDouble(shardLoad -> shardLoad).toArray(), + shardsUptimeInMillis.stream().mapToLong(shardUptime -> shardUptime).toArray() + ); + } + + @Nullable + public static IndexWriteLoad fromStats(IndexMetadata indexMetadata, @Nullable IndicesStatsResponse indicesStatsResponse) { + if (indicesStatsResponse == null) { + return null; + } + + final IndexStats indexStats = indicesStatsResponse.getIndex(indexMetadata.getIndex().getName()); + if (indexStats == null) { + return null; + } + + final int numberOfShards = indexMetadata.getNumberOfShards(); + final var indexWriteLoadBuilder = IndexWriteLoad.builder(numberOfShards); + final var indexShards = indexStats.getIndexShards(); + for (IndexShardStats indexShardsStats : indexShards.values()) { + final var shardStats = Arrays.stream(indexShardsStats.getShards()) + .filter(stats -> stats.getShardRouting().primary()) + .findFirst() + // Fallback to a replica if for some reason we couldn't find the primary stats + .orElse(indexShardsStats.getAt(0)); + final var indexingShardStats = shardStats.getStats().getIndexing().getTotal(); + indexWriteLoadBuilder.withShardWriteLoad( + shardStats.getShardRouting().id(), + indexingShardStats.getWriteLoad(), + indexingShardStats.getTotalActiveTimeInMillis() + ); + } + return indexWriteLoadBuilder.build(); + } + + private final double[] shardWriteLoad; + private final long[] shardUptimeInMillis; + + private IndexWriteLoad(double[] shardWriteLoad, long[] shardUptimeInMillis) { + assert shardWriteLoad.length == shardUptimeInMillis.length; + this.shardWriteLoad = shardWriteLoad; + this.shardUptimeInMillis = shardUptimeInMillis; + } + + public IndexWriteLoad(StreamInput in) throws IOException { + this(in.readDoubleArray(), in.readLongArray()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDoubleArray(shardWriteLoad); + out.writeLongArray(shardUptimeInMillis); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(SHARDS_WRITE_LOAD_FIELD.getPreferredName(), shardWriteLoad); + builder.field(SHARDS_UPTIME_IN_MILLIS.getPreferredName(), shardUptimeInMillis); + return builder; + } + + public static IndexWriteLoad fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + public OptionalDouble getWriteLoadForShard(int shardId) { + assertShardInBounds(shardId); + + double load = shardWriteLoad[shardId]; + return load != UNKNOWN_LOAD ? OptionalDouble.of(load) : OptionalDouble.empty(); + } + + public OptionalLong getUptimeInMillisForShard(int shardId) { + assertShardInBounds(shardId); + + long uptime = shardUptimeInMillis[shardId]; + return uptime != UNKNOWN_UPTIME ? OptionalLong.of(uptime) : OptionalLong.empty(); + } + + // Visible for testing + public int numberOfShards() { + return shardWriteLoad.length; + } + + private void assertShardInBounds(int shardId) { + assert shardId >= 0 : "Unexpected shard id " + shardId; + assert shardId < shardWriteLoad.length : "Unexpected shard id " + shardId + ", expected < " + shardWriteLoad.length; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IndexWriteLoad that = (IndexWriteLoad) o; + return Arrays.equals(shardWriteLoad, that.shardWriteLoad) && Arrays.equals(shardUptimeInMillis, that.shardUptimeInMillis); + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(shardWriteLoad); + result = 31 * result + Arrays.hashCode(shardUptimeInMillis); + return result; + } + + public static Builder builder(int numShards) { + assert numShards > 0 : "A positive number of shards should be provided"; + return new Builder(numShards); + } + + public static class Builder { + final double[] shardWriteLoad; + final long[] uptimeInMillis; + + private Builder(int numShards) { + this.shardWriteLoad = new double[numShards]; + this.uptimeInMillis = new long[numShards]; + Arrays.fill(shardWriteLoad, UNKNOWN_LOAD); + Arrays.fill(uptimeInMillis, UNKNOWN_UPTIME); + } + + public void withShardWriteLoad(int shardId, double load, long uptimeInMillis) { + if (shardId >= this.shardWriteLoad.length) { + throw new IllegalArgumentException(); + } + + this.shardWriteLoad[shardId] = load; + this.uptimeInMillis[shardId] = uptimeInMillis; + } + + public IndexWriteLoad build() { + return new IndexWriteLoad(shardWriteLoad, uptimeInMillis); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java index dd916256a65df..b998b214cef4a 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.concurrent.TimeUnit; public class IndexingStats implements Writeable, ToXContentFragment { @@ -38,7 +39,8 @@ public static class Stats implements Writeable, ToXContentFragment { private long noopUpdateCount; private long throttleTimeInMillis; private boolean isThrottled; - private double writeLoad; + private long totalIndexingTimeSinceShardStartedInNanos; + private long totalActiveTimeInNanos; Stats() {} @@ -54,7 +56,8 @@ public Stats(StreamInput in) throws IOException { isThrottled = in.readBoolean(); throttleTimeInMillis = in.readLong(); if (in.getVersion().onOrAfter(WRITE_LOAD_AVG_SUPPORTED_VERSION)) { - writeLoad = in.readDouble(); + totalIndexingTimeSinceShardStartedInNanos = in.readLong(); + totalActiveTimeInNanos = in.readLong(); } } @@ -69,7 +72,8 @@ public Stats( long noopUpdateCount, boolean isThrottled, long throttleTimeInMillis, - double writeLoad + long totalIndexingTimeSinceShardStartedInNanos, + long totalActiveTimeInNanos ) { this.indexCount = indexCount; this.indexTimeInMillis = indexTimeInMillis; @@ -81,7 +85,9 @@ public Stats( this.noopUpdateCount = noopUpdateCount; this.isThrottled = isThrottled; this.throttleTimeInMillis = throttleTimeInMillis; - this.writeLoad = writeLoad; + // We store the raw write-load values in order to avoid losing precision when we combine the shard stats + this.totalIndexingTimeSinceShardStartedInNanos = totalIndexingTimeSinceShardStartedInNanos; + this.totalActiveTimeInNanos = totalActiveTimeInNanos; } public void add(Stats stats) { @@ -99,7 +105,8 @@ public void add(Stats stats) { if (isThrottled != stats.isThrottled) { isThrottled = true; // When combining if one is throttled set result to throttled. } - writeLoad += stats.writeLoad; + totalIndexingTimeSinceShardStartedInNanos += stats.totalIndexingTimeSinceShardStartedInNanos; + totalActiveTimeInNanos += stats.totalActiveTimeInNanos; } /** @@ -170,7 +177,11 @@ public long getNoopUpdateCount() { } public double getWriteLoad() { - return writeLoad; + return totalActiveTimeInNanos > 0 ? (double) totalIndexingTimeSinceShardStartedInNanos / totalActiveTimeInNanos : 0; + } + + public long getTotalActiveTimeInMillis() { + return TimeUnit.NANOSECONDS.toMillis(totalActiveTimeInNanos); } @Override @@ -186,7 +197,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isThrottled); out.writeLong(throttleTimeInMillis); if (out.getVersion().onOrAfter(WRITE_LOAD_AVG_SUPPORTED_VERSION)) { - out.writeDouble(writeLoad); + out.writeLong(totalIndexingTimeSinceShardStartedInNanos); + out.writeLong(totalActiveTimeInNanos); } } @@ -206,7 +218,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.IS_THROTTLED, isThrottled); builder.humanReadableField(Fields.THROTTLED_TIME_IN_MILLIS, Fields.THROTTLED_TIME, getThrottleTime()); - builder.field(Fields.WRITE_LOAD, writeLoad); + builder.field(Fields.WRITE_LOAD, getWriteLoad()); return builder; } @@ -225,7 +237,8 @@ public boolean equals(Object o) { && noopUpdateCount == that.noopUpdateCount && isThrottled == that.isThrottled && throttleTimeInMillis == that.throttleTimeInMillis - && writeLoad == that.writeLoad; + && totalIndexingTimeSinceShardStartedInNanos == that.totalIndexingTimeSinceShardStartedInNanos + && totalActiveTimeInNanos == that.totalActiveTimeInNanos; } @Override @@ -241,7 +254,8 @@ public int hashCode() { noopUpdateCount, isThrottled, throttleTimeInMillis, - writeLoad + totalIndexingTimeSinceShardStartedInNanos, + totalActiveTimeInNanos ); } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java index 5ed0a21116bec..b0119e23efedc 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java @@ -133,7 +133,7 @@ IndexingStats.Stats stats( long timeSinceShardStartedInNanos ) { final long totalIndexingTimeInNanos = indexMetric.sum(); - final long totalIndexingTimeSinceShardStarted = totalIndexingTimeInNanos - indexingTimeBeforeShardStartedInNanos; + final long totalIndexingTimeSinceShardStartedInNanos = totalIndexingTimeInNanos - indexingTimeBeforeShardStartedInNanos; return new IndexingStats.Stats( indexMetric.count(), TimeUnit.NANOSECONDS.toMillis(totalIndexingTimeInNanos), @@ -145,7 +145,8 @@ IndexingStats.Stats stats( noopUpdates.count(), isThrottled, TimeUnit.MILLISECONDS.toMillis(currentThrottleMillis), - timeSinceShardStartedInNanos > 0 ? (double) totalIndexingTimeSinceShardStarted / timeSinceShardStartedInNanos : 0 + totalIndexingTimeSinceShardStartedInNanos, + timeSinceShardStartedInNanos ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 97cfc71da3a4a..6b8a38e372e94 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -575,6 +575,7 @@ private static CommonStats createShardLevelCommonStats() { ++iota, false, ++iota, + ++iota, ++iota ); indicesCommonStats.getIndexing().add(new IndexingStats(indexingStats)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index d7eb90b1d3187..e95eae894fc6b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -546,7 +546,8 @@ public void testRolloverClusterState() throws Exception { metConditions, Instant.now(), randomBoolean(), - false + false, + null ); long after = testThreadPool.absoluteTimeInMillis(); @@ -612,7 +613,8 @@ public void testRolloverClusterStateForDataStream() throws Exception { metConditions, Instant.now(), randomBoolean(), - false + false, + null ); long after = testThreadPool.absoluteTimeInMillis(); @@ -695,7 +697,8 @@ public void testValidation() throws Exception { null, Instant.now(), randomBoolean(), - true + true, + null ); newIndexName = newIndexName == null ? defaultRolloverIndexName : newIndexName; @@ -735,7 +738,8 @@ public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception { metConditions, Instant.now(), false, - randomBoolean() + randomBoolean(), + null ) ); assertThat(e.getMessage(), equalTo("no matching index template found for data stream [" + dataStream.getName() + "]")); diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 31cd19b5ccbc8..f76b223884703 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; @@ -281,6 +282,10 @@ public void testToXContent() throws IOException { "system": false, "timestamp_range": { "shards": [] + }, + "write_load": { + "loads": [-1.0], + "uptimes": [-1] } } }, @@ -493,6 +498,14 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "system" : false, "timestamp_range" : { "shards" : [ ] + }, + "write_load" : { + "loads" : [ + -1.0 + ], + "uptimes" : [ + -1 + ] } } }, @@ -712,6 +725,14 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "system" : false, "timestamp_range" : { "shards" : [ ] + }, + "write_load" : { + "loads" : [ + -1.0 + ], + "uptimes" : [ + -1 + ] } } }, @@ -901,6 +922,7 @@ private ClusterState buildClusterState() throws IOException { }) .numberOfReplicas(2) .putRolloverInfo(new RolloverInfo("rolloveAlias", new ArrayList<>(), 1L)) + .indexWriteLoad(IndexWriteLoad.builder(1).build()) .build(); return ClusterState.builder(ClusterName.DEFAULT) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index e85e159cf1220..0c2568491625e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.shard.IndexWriteLoad; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.ESTestCase; @@ -72,6 +73,7 @@ public void testIndexMetadataSerialization() throws IOException { Map customMap = new HashMap<>(); customMap.put(randomAlphaOfLength(5), randomAlphaOfLength(10)); customMap.put(randomAlphaOfLength(10), randomAlphaOfLength(15)); + IndexWriteLoad indexWriteLoad = randomBoolean() ? randomWriteLoad(numShard) : null; IndexMetadata metadata = IndexMetadata.builder("foo") .settings( Settings.builder() @@ -98,6 +100,7 @@ public void testIndexMetadataSerialization() throws IOException { randomNonNegativeLong() ) ) + .indexWriteLoad(indexWriteLoad) .build(); assertEquals(system, metadata.isSystem()); @@ -126,6 +129,7 @@ public void testIndexMetadataSerialization() throws IOException { Map expectedCustom = Map.of("my_custom", new DiffableStringMap(customMap)); assertEquals(metadata.getCustomData(), expectedCustom); assertEquals(metadata.getCustomData(), fromXContentMeta.getCustomData()); + assertEquals(metadata.getWriteLoad(), fromXContentMeta.getWriteLoad()); final BytesStreamOutput out = new BytesStreamOutput(); metadata.writeTo(out); @@ -146,6 +150,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(deserialized.getCustomData(), expectedCustom); assertEquals(metadata.getCustomData(), deserialized.getCustomData()); assertEquals(metadata.isSystem(), deserialized.isSystem()); + assertEquals(metadata.getWriteLoad(), deserialized.getWriteLoad()); } } @@ -490,4 +495,13 @@ private static Settings indexSettingsWithDataTier(String dataTier) { .put(DataTier.TIER_PREFERENCE, dataTier) .build(); } + + private IndexWriteLoad randomWriteLoad(int numberOfShards) { + IndexWriteLoad.Builder indexWriteLoadBuilder = IndexWriteLoad.builder(numberOfShards); + int numberOfPopulatedWriteLoads = randomIntBetween(0, numberOfShards); + for (int i = 0; i < numberOfPopulatedWriteLoads; i++) { + indexWriteLoadBuilder.withShardWriteLoad(i, randomDoubleBetween(0.0, 128.0, true), randomNonNegativeLong()); + } + return indexWriteLoadBuilder.build(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadSerializationTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadSerializationTests.java new file mode 100644 index 0000000000000..1d74b9fca0617 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadSerializationTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +public class IndexWriteLoadSerializationTests extends AbstractXContentSerializingTestCase { + + @Override + protected IndexWriteLoad doParseInstance(XContentParser parser) throws IOException { + return IndexWriteLoad.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return IndexWriteLoad::new; + } + + @Override + protected IndexWriteLoad createTestInstance() { + final int numberOfShards = randomIntBetween(1, 10); + final var indexWriteLoad = IndexWriteLoad.builder(numberOfShards); + for (int i = 0; i < numberOfShards; i++) { + indexWriteLoad.withShardWriteLoad(i, randomDoubleBetween(1, 10, true), randomLongBetween(1, 1000)); + } + return indexWriteLoad.build(); + } + + @Override + protected IndexWriteLoad mutateInstance(IndexWriteLoad instance) throws IOException { + final int newNumberOfShards; + if (instance.numberOfShards() > 1 && randomBoolean()) { + newNumberOfShards = randomIntBetween(1, instance.numberOfShards() - 1); + } else { + newNumberOfShards = instance.numberOfShards() + randomIntBetween(1, 5); + } + final var indexWriteLoad = IndexWriteLoad.builder(newNumberOfShards); + for (int i = 0; i < newNumberOfShards; i++) { + boolean existingShard = i < instance.numberOfShards(); + double shardLoad = existingShard && randomBoolean() + ? instance.getWriteLoadForShard(i).getAsDouble() + : randomDoubleBetween(0, 128, true); + long uptimeInMillis = existingShard && randomBoolean() + ? instance.getUptimeInMillisForShard(i).getAsLong() + : randomNonNegativeLong(); + indexWriteLoad.withShardWriteLoad(i, shardLoad, uptimeInMillis); + } + return indexWriteLoad.build(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadTests.java new file mode 100644 index 0000000000000..bf119f1560335 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexWriteLoadTests.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingHelper; +import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class IndexWriteLoadTests extends ESTestCase { + + public void testGetWriteLoadForShardAndGetUptimeInMillisForShard() { + final int numberOfPopulatedShards = 10; + final int numberOfShards = randomIntBetween(numberOfPopulatedShards, 20); + final IndexWriteLoad.Builder indexWriteLoadBuilder = IndexWriteLoad.builder(numberOfShards); + + final double[] populatedShardWriteLoads = new double[numberOfPopulatedShards]; + final long[] populatedShardUptimes = new long[numberOfPopulatedShards]; + for (int shardId = 0; shardId < numberOfPopulatedShards; shardId++) { + double writeLoad = randomDoubleBetween(1, 128, true); + long uptimeInMillis = randomNonNegativeLong(); + populatedShardWriteLoads[shardId] = writeLoad; + populatedShardUptimes[shardId] = uptimeInMillis; + indexWriteLoadBuilder.withShardWriteLoad(shardId, writeLoad, uptimeInMillis); + } + + final IndexWriteLoad indexWriteLoad = indexWriteLoadBuilder.build(); + for (int shardId = 0; shardId < numberOfShards; shardId++) { + if (shardId < numberOfPopulatedShards) { + assertThat(indexWriteLoad.getWriteLoadForShard(shardId).isPresent(), is(equalTo(true))); + assertThat(indexWriteLoad.getWriteLoadForShard(shardId).getAsDouble(), is(equalTo(populatedShardWriteLoads[shardId]))); + assertThat(indexWriteLoad.getUptimeInMillisForShard(shardId).isPresent(), is(equalTo(true))); + assertThat(indexWriteLoad.getUptimeInMillisForShard(shardId).getAsLong(), is(equalTo(populatedShardUptimes[shardId]))); + } else { + assertThat(indexWriteLoad.getWriteLoadForShard(shardId).isPresent(), is(false)); + assertThat(indexWriteLoad.getUptimeInMillisForShard(shardId).isPresent(), is(false)); + } + } + } + + public void testFromStatsCreation() { + final String indexName = "idx"; + final IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build() + ) + .build(); + + final IndicesStatsResponse response = mock(IndicesStatsResponse.class); + final IndexStats indexStats = mock(IndexStats.class); + + // Shard 0 has both primary/replica + final IndexShardStats indexShard0Stats = new IndexShardStats( + new ShardId(indexName, "__na__", 0), + new ShardStats[] { + createShardStats(indexName, 0, true, TimeValue.timeValueMillis(2048).nanos(), TimeValue.timeValueMillis(1024).nanos()), + createShardStats(indexName, 0, false, TimeValue.timeValueMillis(2048).nanos(), TimeValue.timeValueMillis(512).nanos()) } + ); + + // Shard 1 only has a replica available + final IndexShardStats indexShard1Stats = new IndexShardStats( + new ShardId(indexName, "__na__", 1), + new ShardStats[] { + createShardStats(indexName, 1, false, TimeValue.timeValueMillis(4096).nanos(), TimeValue.timeValueMillis(512).nanos()) } + ); + // Shard 2 was not available + + when(response.getIndex(indexName)).thenReturn(indexStats); + when(indexStats.getIndexShards()).thenReturn(Map.of(0, indexShard0Stats, 1, indexShard1Stats)); + + // Shard 0 uses the results from the primary + final IndexWriteLoad indexWriteLoadFromStats = IndexWriteLoad.fromStats(indexMetadata, response); + assertThat(indexWriteLoadFromStats.getWriteLoadForShard(0).isPresent(), is(equalTo(true))); + assertThat(indexWriteLoadFromStats.getWriteLoadForShard(0).getAsDouble(), is(equalTo(2.0))); + assertThat(indexWriteLoadFromStats.getUptimeInMillisForShard(0).isPresent(), is(equalTo(true))); + assertThat(indexWriteLoadFromStats.getUptimeInMillisForShard(0).getAsLong(), is(equalTo(1024L))); + + // Shard 1 uses the only available stats from a replica + assertThat(indexWriteLoadFromStats.getWriteLoadForShard(1).isPresent(), is(equalTo(true))); + assertThat(indexWriteLoadFromStats.getWriteLoadForShard(1).getAsDouble(), is(equalTo(8.0))); + assertThat(indexWriteLoadFromStats.getUptimeInMillisForShard(1).isPresent(), is(equalTo(true))); + assertThat(indexWriteLoadFromStats.getUptimeInMillisForShard(1).getAsLong(), is(equalTo(512L))); + + assertThat(indexWriteLoadFromStats.getWriteLoadForShard(2).isPresent(), is(equalTo(false))); + assertThat(indexWriteLoadFromStats.getUptimeInMillisForShard(2).isPresent(), is(equalTo(false))); + + assertThat(IndexWriteLoad.fromStats(indexMetadata, null), is(nullValue())); + } + + private ShardStats createShardStats( + String indexName, + int shard, + boolean primary, + long totalIndexingTimeSinceShardStartedInNanos, + long totalActiveTimeInNanos + ) { + RecoverySource recoverySource = primary + ? RecoverySource.EmptyStoreRecoverySource.INSTANCE + : RecoverySource.PeerRecoverySource.INSTANCE; + ShardRouting shardRouting = ShardRouting.newUnassigned( + new ShardId(indexName, "__na__", shard), + primary, + recoverySource, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo") + ); + shardRouting = ShardRoutingHelper.initialize(shardRouting, UUIDs.randomBase64UUID()); + shardRouting = ShardRoutingHelper.moveToStarted(shardRouting); + + final CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL); + commonStats.getIndexing() + .getTotal() + .add( + new IndexingStats.Stats(0, 0, 0, 0, 0, 0, 0, 0, false, 0, totalIndexingTimeSinceShardStartedInNanos, totalActiveTimeInNanos) + ); + return new ShardStats(shardRouting, commonStats, null, null, null, null, null, false); + } +} diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java index 0654025af1b7e..e9e2a19df690a 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java @@ -386,7 +386,7 @@ private static CommonStats mockCommonStats() { commonStats.getStore().add(new StoreStats(++iota, no, no)); commonStats.getRefresh().add(new RefreshStats(no, ++iota, no, ++iota, (int) no)); - final IndexingStats.Stats indexingStats = new IndexingStats.Stats(++iota, ++iota, no, no, no, no, no, no, false, ++iota, no); + final IndexingStats.Stats indexingStats = new IndexingStats.Stats(++iota, ++iota, no, no, no, no, no, no, false, ++iota, no, no); commonStats.getIndexing().add(new IndexingStats(indexingStats)); final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java index 502391889a0e7..8ae1c79d3b3f4 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java @@ -183,7 +183,7 @@ private CommonStats mockCommonStats() { commonStats.getDocs().add(new DocsStats(1L, 0L, randomNonNegativeLong())); commonStats.getStore().add(new StoreStats(2L, 0L, 0L)); - final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, 0L, 0L, 0L, 0L, 0L, 0L, true, 5L, 0); + final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, 0L, 0L, 0L, 0L, 0L, 0L, true, 5L, 0, 0); commonStats.getIndexing().add(new IndexingStats(indexingStats)); final SearchStats.Stats searchStats = new SearchStats.Stats(6L, 7L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java index 8b3abd7fc5bf8..f270d23161c5b 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java @@ -328,7 +328,20 @@ private static NodeStats mockNodeStats() { indicesCommonStats.getFieldData().add(new FieldDataStats(++iota, ++iota, null)); indicesCommonStats.getStore().add(new StoreStats(++iota, no, no)); - final IndexingStats.Stats indexingStats = new IndexingStats.Stats(++iota, ++iota, ++iota, no, no, no, no, no, false, ++iota, no); + final IndexingStats.Stats indexingStats = new IndexingStats.Stats( + ++iota, + ++iota, + ++iota, + no, + no, + no, + no, + no, + false, + ++iota, + no, + no + ); indicesCommonStats.getIndexing().add(new IndexingStats(indexingStats)); indicesCommonStats.getQueryCache().add(new QueryCacheStats(++iota, ++iota, ++iota, ++iota, no)); indicesCommonStats.getRequestCache().add(new RequestCacheStats(++iota, ++iota, ++iota, ++iota)); From 9e830840209a7543417f48bb375e661950d68754 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 4 Nov 2022 08:15:46 -0700 Subject: [PATCH 13/17] [DOCS] Clarify description of geo_results (#91237) --- .../apis/get-record.asciidoc | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc index 16c3c3705c19f..a3a72bc39ab73 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc @@ -190,15 +190,19 @@ configuration. For example, `max`. (string) The description of the function in which the anomaly occurs, as specified in the detector configuration. -`geo_results.actual_point`:: -(string) The actual value for the bucket formatted as a `geo_point`. If the -detector function is `lat_long`, this is a comma delimited string of the -latitude and longitude. - -`geo_results.typical_point`:: -(string) The typical value for the bucket formatted as a `geo_point`. If the -detector function is `lat_long`, this is a comma delimited string of the -latitude and longitude. +`geo_results`:: +(optional, object) If the detector function is `lat_long`, this object contains +comma delimited strings for the latitude and longitude of the actual and typical values. ++ +.Properties of `geo_results` +[%collapsible%open] +==== +`actual_point`:: +(string) The actual value for the bucket formatted as a `geo_point`. + +`typical_point`:: +(string) The typical value for the bucket formatted as a `geo_point`. +==== `influencers`:: (array) If `influencers` was specified in the detector configuration, this array From 0b689b744a303b9413cb6133fa64112e2ed90b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Fri, 4 Nov 2022 16:25:52 +0100 Subject: [PATCH 14/17] Fix SnapshotBasedIndexRecoveryIT#testNodeDisconnectsDoNotOverAccountRecoveredBytes (#90849) The clock used to check for timeouts does not provide enough granularity, and sometimes the RESTORE_FILE_FROM_SNAPSHOT is retried while the test expects to fail right away. This commit configures the nodes with a timeout of 0, effectively configuring it to avoid retrying recovery requests. Closes #90665 --- .../SnapshotBasedIndexRecoveryIT.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java index 3f8b33158437f..431a7d5c7a5d5 100644 --- a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java +++ b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java @@ -100,6 +100,7 @@ import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_RETRY_TIMEOUT_SETTING; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING; +import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_SNAPSHOT_FILE_DOWNLOADS; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_SNAPSHOT_FILE_DOWNLOADS_PER_NODE; @@ -1255,7 +1256,6 @@ public void testRecoveryRetryKeepsTheGrantedSnapshotFileDownloadPermit() throws ); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/90665") public void testNodeDisconnectsDoNotOverAccountRecoveredBytes() throws Exception { // This test reproduces a rare (but possible scenario) where a shard is recovering using // snapshots, using logically equivalent index files, but half-way the connection between @@ -1266,14 +1266,7 @@ public void testNodeDisconnectsDoNotOverAccountRecoveredBytes() throws Exception // - The target updates the recovered bytes for the file it has been downloading, after the recovery state was cleared. // This could end up over-accounting the number of recovered bytes - Settings nodeSettings = Settings.builder() - // We use a really low timeout to avoid retrying the first RESTORE_FILE_FROM_SNAPSHOT after the connection is dropped - .put(INDICES_RECOVERY_INTERNAL_ACTION_RETRY_TIMEOUT_SETTING.getKey(), TimeValue.timeValueMillis(1)) - .put(INDICES_RECOVERY_MAX_CONCURRENT_SNAPSHOT_FILE_DOWNLOADS.getKey(), 1) - .put(INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING.getKey(), 1) - .build(); - - List dataNodes = internalCluster().startDataOnlyNodes(3, nodeSettings); + List dataNodes = internalCluster().startDataOnlyNodes(3); String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); createIndex( indexName, @@ -1287,6 +1280,21 @@ public void testNodeDisconnectsDoNotOverAccountRecoveredBytes() throws Exception ); ensureGreen(indexName); + assertAcked( + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings( + Settings.builder() + // Do not retry the first RESTORE_FILE_FROM_SNAPSHOT after the connection is closed + .put(INDICES_RECOVERY_INTERNAL_ACTION_RETRY_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) + .put(INDICES_RECOVERY_MAX_CONCURRENT_SNAPSHOT_FILE_DOWNLOADS.getKey(), 1) + .put(INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING.getKey(), 1) + .put(INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING.getKey(), 1) + .build() + ) + ); + int numDocs = randomIntBetween(300, 1000); indexDocs(indexName, 0, numDocs); // Flush to ensure that index_commit_seq_nos(replica) == index_commit_seq_nos(primary), From bdf34852576159b3919b94979559bee5444f425c Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 4 Nov 2022 15:27:53 +0000 Subject: [PATCH 15/17] Revert "Skip prevoting if single-node discovery (#91255)" This reverts commit 611c9b86e2b81edaf0aaf936f79d5c4e913786a3. --- .../org/elasticsearch/cluster/coordination/Coordinator.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 4844d079c46ef..33e3528246d12 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -1628,10 +1628,6 @@ public void run() { } if (prevotingRound != null) { - if (singleNodeDiscovery) { - logger.debug("skip prevoting as single-node discovery is enabled"); - return; - } prevotingRound.close(); } prevotingRound = preVoteCollector.start(lastAcceptedState, getDiscoveredNodes()); From cd226cc78a0b965ebfb26536cd42b3afa1bcd2aa Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 4 Nov 2022 15:37:04 +0000 Subject: [PATCH 16/17] AwaitsFix for 89867 --- .../org/elasticsearch/cluster/coordination/CoordinatorTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 82c792622eeab..ebcac01df64a0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -1904,6 +1904,7 @@ public void testSingleNodeDiscoveryWithQuorum() { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/89867") public void testSingleNodeDiscoveryStabilisesEvenWhenDisrupted() { try ( Cluster cluster = new Cluster( From af537cc4a377613ba7c018eea8b3bcd27a6c1fa9 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 4 Nov 2022 18:11:12 +0200 Subject: [PATCH 17/17] Fix index expression options for requests with a single expression (#91231) This PR affects requests that contain a single index name or a single pattern (wildcard/datemath). It aims to systematize the handling of the `allow_no_indices` and `ignore_unavailable`indices options: * the allow_no_indices option is to be concerned with wildcards that expand to nothing (or the entire request expands to nothing) * the ignore_unavailable option is to be concerned with explicit names only (not wildcards) In addition, the behavior of the above options will now be independent of the number of expressions in a request. --- docs/changelog/91231.yaml | 5 ++ .../index/rankeval/RankEvalRequestIT.java | 2 +- .../50_wildcard_expansion.yml | 14 +++++ .../validate/SimpleValidateQueryIT.java | 2 +- .../metadata/IndexNameExpressionResolver.java | 56 +++++++++---------- .../IndexNameExpressionResolverTests.java | 25 +++++---- 6 files changed, 61 insertions(+), 43 deletions(-) create mode 100644 docs/changelog/91231.yaml diff --git a/docs/changelog/91231.yaml b/docs/changelog/91231.yaml new file mode 100644 index 0000000000000..1ca88ff84d8dc --- /dev/null +++ b/docs/changelog/91231.yaml @@ -0,0 +1,5 @@ +pr: 91231 +summary: Fix index expression options for requests with a single name or pattern +area: Infra/Core +type: bug +issues: [] diff --git a/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 8dfc592b6f4a5..64a26c426ff55 100644 --- a/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -291,7 +291,7 @@ public void testIndicesOptions() { // test expand_wildcards request = new RankEvalRequest(task, new String[] { "tes*" }); - request.indicesOptions(IndicesOptions.fromParameters("none", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters("none", "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(0, details.getRetrieved()); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml index 5069a85f16e69..c6ff83345c6a0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml @@ -111,6 +111,7 @@ setup: indices.get_mapping: index: test-x* expand_wildcards: none + ignore_unavailable: true - match: { '': {} } --- @@ -121,6 +122,19 @@ setup: index: test-x* expand_wildcards: none allow_no_indices: false + ignore_unavailable: true +--- +"Get test-* with wildcard_expansion=none ignore_unavailable=false": + - skip: + version: " - 8.5.99" + reason: "bug fixed in 8.6" + - do: + catch: missing + indices.get_mapping: + index: test-x* + expand_wildcards: none + allow_no_indices: true + ignore_unavailable: false --- "Get test-* with wildcard_expansion=open,closed": diff --git a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index b3ff6688aa8c6..29cd365ff403c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -260,7 +260,7 @@ public void testValidateEmptyCluster() { client().admin().indices().prepareValidateQuery().get(); fail("Expected IndexNotFoundException"); } catch (IndexNotFoundException e) { - assertThat(e.getMessage(), is("no such index [null] and no indices exist")); + assertThat(e.getMessage(), is("no such index [_all] and no indices exist")); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 19fae95341396..b319aebe18caf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -335,28 +335,12 @@ Index[] concreteIndices(Context context, String... indexExpressions) { } } } - // If only one index is specified then whether we fail a request if an index is missing depends on the allow_no_indices - // option. At some point we should change this, because there shouldn't be a reason why whether a single index - // or multiple indices are specified yield different behaviour. - final boolean failNoIndices = indexExpressions.length == 1 - ? options.allowNoIndices() == false - : options.ignoreUnavailable() == false; + final Collection expressions = resolveExpressions(Arrays.asList(indexExpressions), context); - if (expressions.isEmpty()) { + if (expressions.isEmpty() || (expressions.size() == 1 && expressions.iterator().next().equals(Metadata.ALL))) { if (options.allowNoIndices() == false) { - IndexNotFoundException infe; - if (indexExpressions.length == 1) { - if (indexExpressions[0].equals(Metadata.ALL)) { - infe = new IndexNotFoundException("no indices exist", (String) null); - } else { - infe = new IndexNotFoundException((String) null); - } - } else { - infe = new IndexNotFoundException((String) null); - } - infe.setResources("index_expression", indexExpressions); - throw infe; + throw notFoundException(indexExpressions); } else { return Index.EMPTY_ARRAY; } @@ -368,20 +352,15 @@ Index[] concreteIndices(Context context, String... indexExpressions) { for (String expression : expressions) { IndexAbstraction indexAbstraction = indicesLookup.get(expression); if (indexAbstraction == null) { - if (failNoIndices) { - IndexNotFoundException infe; - if (expression.equals(Metadata.ALL)) { - infe = new IndexNotFoundException("no indices exist", expression); - } else { - infe = new IndexNotFoundException(expression); - } - infe.setResources("index_expression", expression); - throw infe; + if (options.ignoreUnavailable() == false) { + assert options.expandWildcardExpressions() == false; + throw notFoundException(expression); } else { continue; } } else if (indexAbstraction.getType() == Type.ALIAS && context.getOptions().ignoreAliases()) { - if (failNoIndices) { + if (options.ignoreUnavailable() == false) { + assert options.expandWildcardExpressions() == false; throw aliasesNotSupportedException(expression); } else { continue; @@ -436,8 +415,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) { } if (options.allowNoIndices() == false && concreteIndices.isEmpty()) { - IndexNotFoundException infe = new IndexNotFoundException((String) null); - infe.setResources("index_expression", indexExpressions); + IndexNotFoundException infe = notFoundException(indexExpressions); if (excludedDataStreams) { // Allows callers to handle IndexNotFoundException differently based on whether data streams were excluded. infe.addMetadata(EXCLUDED_DATA_STREAMS_KEY, "true"); @@ -448,6 +426,22 @@ Index[] concreteIndices(Context context, String... indexExpressions) { return concreteIndices.toArray(Index.EMPTY_ARRAY); } + private IndexNotFoundException notFoundException(String... indexExpressions) { + IndexNotFoundException infe; + if (indexExpressions.length == 1) { + if (indexExpressions[0].equals(Metadata.ALL)) { + infe = new IndexNotFoundException("no indices exist", indexExpressions[0]); + } else { + infe = new IndexNotFoundException(indexExpressions[0]); + } + infe.setResources("index_expression", indexExpressions[0]); + } else { + infe = new IndexNotFoundException((String) null); + infe.setResources("index_expression", indexExpressions); + } + return infe; + } + private void checkSystemIndexAccess(Context context, Set concreteIndices) { final Metadata metadata = context.getState().metadata(); final Predicate systemIndexAccessPredicate = context.getSystemIndexAccessPredicate().negate(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 0aa6bde7008b1..d08166eaf4118 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -478,8 +478,9 @@ public void testConcreteIndexNamesExpandWildcards() { results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY); assertThat(results, emptyArray()); - results = indexNameExpressionResolver.concreteIndexNames(context, "h*"); - assertThat(results, emptyArray()); + IndexNameExpressionResolver.Context context3 = context; + infe = expectThrows(IndexNotFoundException.class, () -> indexNameExpressionResolver.concreteIndexNames(context3, "h*")); + assertThat(infe.getResourceId().toString(), equalTo("[h*]")); results = indexNameExpressionResolver.concreteIndexNames(context, "hidden"); assertThat(results, arrayContainingInAnyOrder("hidden")); @@ -562,8 +563,11 @@ public void testConcreteIndexNamesNoExpandWildcards() { SystemIndexAccessLevel.NONE ); { - String[] results = indexNameExpressionResolver.concreteIndexNames(context, "baz*"); - assertThat(results, emptyArray()); + IndexNotFoundException infe = expectThrows( + IndexNotFoundException.class, + () -> indexNameExpressionResolver.concreteIndexNames(context, "baz*") + ); + assertThat(infe.getIndex().getName(), equalTo("baz*")); } { IndexNotFoundException infe = expectThrows( @@ -829,7 +833,7 @@ public void testConcreteIndicesNoIndicesErrorMessage() { IndexNotFoundException.class, () -> indexNameExpressionResolver.concreteIndices(context, new String[] {}) ); - assertThat(infe.getMessage(), is("no such index [null] and no indices exist")); + assertThat(infe.getMessage(), is("no such index [_all] and no indices exist")); } public void testConcreteIndicesNoIndicesErrorMessageNoExpand() { @@ -1292,13 +1296,14 @@ public void testConcreteIndicesWildcardNoMatch() { SystemIndexAccessLevel.NONE ); - // asking for non existing wildcard pattern should return empty list or exception - if (indicesOptions.allowNoIndices()) { + if (indicesOptions.allowNoIndices() == false + || indicesOptions.expandWildcardExpressions() == false && indicesOptions.ignoreUnavailable() == false) { + expectThrows(IndexNotFoundException.class, () -> indexNameExpressionResolver.concreteIndexNames(context, "Foo*")); + } else { + // asking for non existing wildcard pattern should return empty list or exception String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(context, "Foo*"); assertThat(concreteIndices, notNullValue()); assertThat(concreteIndices.length, equalTo(0)); - } else { - expectThrows(IndexNotFoundException.class, () -> indexNameExpressionResolver.concreteIndexNames(context, "Foo*")); } } } @@ -2471,7 +2476,7 @@ public void testDataStreams() { IndexNotFoundException.class, () -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false, false) ); - assertThat(e.getMessage(), equalTo("no such index [null]")); + assertThat(e.getMessage(), equalTo("no such index [my-data-stream]")); } }