From 83262e84f772f18f39428d8f3f9eeb190f7f1377 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 15 Oct 2021 14:45:31 +0200 Subject: [PATCH] Refactor IndexAbstraction to not use IndexMetadata Backport of #79080 to 7.x branch. Most users of an `IndexAbstraction` instance doesn't need to use the `IndexMetadata` instances that `getIndices()` and `getWriteIndex()` returns. Cluster state variables/parameters can be used in places that access to `IndexMetadata` is required. By changing the `getIndices()` and `getWriteIndex()` methods to return `Index` instance, the indices lookup can be reused across different cluster states. This should be possible in cases that don't change an index hidden status or open and closes indices or when adding / removing aliases, data streams or indices. This change should allow for #79004 --- .../elasticsearch/aliases/IndexAliasesIT.java | 5 +- .../indices/resolve/ResolveIndexAction.java | 22 ++--- .../rollover/MetadataRolloverService.java | 21 ++--- .../cluster/metadata/AliasMetadata.java | 5 +- .../cluster/metadata/DataStream.java | 2 +- .../cluster/metadata/IndexAbstraction.java | 89 ++++++++++--------- .../metadata/IndexAbstractionResolver.java | 2 +- .../metadata/IndexNameExpressionResolver.java | 63 +++++++------ .../cluster/metadata/Metadata.java | 38 ++++---- .../metadata/MetadataCreateIndexService.java | 2 +- .../metadata/MetadataDeleteIndexService.java | 2 +- .../metadata/MetadataIndexStateService.java | 10 ++- .../MetadataMigrateToDataStreamService.java | 19 ++-- .../elasticsearch/ingest/IngestService.java | 8 +- .../mapping/put/PutMappingRequestTests.java | 8 +- .../MetadataRolloverServiceTests.java | 12 +-- .../action/bulk/TransportBulkActionTests.java | 6 +- .../MetadataIndexAliasesServiceTests.java | 22 ++--- ...tadataMigrateToDataStreamServiceTests.java | 15 ++-- .../cluster/metadata/MetadataTests.java | 9 +- .../ReactiveStorageDeciderService.java | 17 ++-- .../ProactiveStorageDeciderServiceTests.java | 5 +- .../ilm/CheckNotDataStreamWriteIndexStep.java | 2 +- .../xpack/core/ilm/DeleteStep.java | 4 +- .../ReplaceDataStreamBackingIndexStep.java | 2 +- .../xpack/core/ilm/RolloverStep.java | 2 +- .../core/ilm/WaitForActiveShardsStep.java | 27 +++--- .../core/ilm/WaitForRolloverReadyStep.java | 2 +- .../persistence/ElasticsearchMappings.java | 4 +- .../xpack/core/ml/utils/MlIndexAndAlias.java | 8 +- .../authz/permission/IndicesPermission.java | 6 +- .../xpack/core/ilm/DeleteStepTests.java | 32 +++++-- .../core/ilm/LifecyclePolicyUtilsTests.java | 70 ++++++++------- .../DataStreamsStatsTransportAction.java | 11 +-- .../xpack/enrich/EnrichCache.java | 2 +- .../xpack/enrich/EnrichProcessorFactory.java | 2 +- .../action/TransportFreezeIndexAction.java | 4 +- .../idp/saml/sp/SamlServiceProviderIndex.java | 2 +- .../authz/IndicesAndAliasesResolver.java | 10 +-- .../xpack/security/authz/RBACEngine.java | 6 +- .../DeprecationRoleDescriptorConsumer.java | 13 ++- .../support/SecurityIndexManager.java | 8 +- .../test/SecurityIntegTestCase.java | 2 +- .../xpack/security/authz/RBACEngineTests.java | 8 +- .../TransformClusterStateListener.java | 4 +- .../xpack/watcher/watch/WatchStoreUtils.java | 2 +- .../watcher/WatcherIndexingListenerTests.java | 3 +- 47 files changed, 329 insertions(+), 289 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java index 7658d603bdeec..f7e8aac162b39 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -668,8 +668,9 @@ public void testSameAlias() throws Exception { assertThat(stopWatch.stop().lastTaskTime().millis(), lessThan(timeout.millis())); logger.info("--> verify that filter was updated"); - IndexAbstraction ia = internalCluster().clusterService().state().metadata().getIndicesLookup().get("alias1"); - AliasMetadata aliasMetadata = AliasMetadata.getFirstAliasMetadata(ia); + Metadata metadata = internalCluster().clusterService().state().metadata(); + IndexAbstraction ia = metadata.getIndicesLookup().get("alias1"); + AliasMetadata aliasMetadata = AliasMetadata.getFirstAliasMetadata(metadata, ia); assertThat(aliasMetadata.getFilter().toString(), equalTo("{\"term\":{\"name\":{\"value\":\"bar\",\"boost\":1.0}}}")); logger.info("--> deleting alias1"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 78cf4f68dab95..ead66843f9dfb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.index.Index; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -506,7 +507,7 @@ static void resolveIndices(String[] names, IndicesOptions indicesOptions, Metada includeDataStreams); SortedMap lookup = metadata.getIndicesLookup(); for (String s : resolvedIndexAbstractions) { - enrichIndexAbstraction(s, lookup, indices, aliases, dataStreams); + enrichIndexAbstraction(metadata, s, lookup, indices, aliases, dataStreams); } indices.sort(Comparator.comparing(ResolvedIndexAbstraction::getName)); aliases.sort(Comparator.comparing(ResolvedIndexAbstraction::getName)); @@ -531,42 +532,41 @@ private static void mergeResults(Map remoteResponses, List lookup, + private static void enrichIndexAbstraction(Metadata metadata, String indexAbstraction, SortedMap lookup, List indices, List aliases, List dataStreams) { IndexAbstraction ia = lookup.get(indexAbstraction); if (ia != null) { switch (ia.getType()) { case CONCRETE_INDEX: - IndexAbstraction.Index index = (IndexAbstraction.Index) ia; - - String[] aliasNames = index.getWriteIndex().getAliases().keySet().stream().sorted().toArray(String[]::new); + IndexMetadata writeIndex = metadata.index(ia.getWriteIndex()); + String[] aliasNames = writeIndex.getAliases().keySet().stream().sorted().toArray(String[]::new); List attributes = new ArrayList<>(); - attributes.add(index.getWriteIndex().getState() == IndexMetadata.State.OPEN ? "open" : "closed"); + attributes.add(writeIndex.getState() == IndexMetadata.State.OPEN ? "open" : "closed"); if (ia.isHidden()) { attributes.add("hidden"); } - final boolean isFrozen = Boolean.parseBoolean(ia.getWriteIndex().getSettings().get("index.frozen")); + final boolean isFrozen = Boolean.parseBoolean(writeIndex.getSettings().get("index.frozen")); if (isFrozen) { attributes.add("frozen"); } attributes.sort(String::compareTo); indices.add(new ResolvedIndex( - index.getName(), + ia.getName(), aliasNames, attributes.toArray(Strings.EMPTY_ARRAY), - index.getParentDataStream() == null ? null : index.getParentDataStream().getName())); + ia.getParentDataStream() == null ? null : ia.getParentDataStream().getName())); break; case ALIAS: - String[] indexNames = ia.getIndices().stream().map(i -> i.getIndex().getName()).toArray(String[]::new); + String[] indexNames = ia.getIndices().stream().map(Index::getName).toArray(String[]::new); Arrays.sort(indexNames); aliases.add(new ResolvedAlias(ia.getName(), indexNames)); break; case DATA_STREAM: IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) ia; - String[] backingIndices = dataStream.getIndices().stream().map(i -> i.getIndex().getName()).toArray(String[]::new); + String[] backingIndices = dataStream.getIndices().stream().map(Index::getName).toArray(String[]::new); dataStreams.add(new ResolvedDataStream( dataStream.getName(), backingIndices, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 46a933dd45421..22484d76584f4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.snapshots.SnapshotInProgressException; @@ -128,7 +129,7 @@ public NameResolution resolveRolloverNames(ClusterState currentState, String rol final IndexAbstraction indexAbstraction = currentState.metadata().getIndicesLookup().get(rolloverTarget); switch (indexAbstraction.getType()) { case ALIAS: - return resolveAliasRolloverNames((IndexAbstraction.Alias) indexAbstraction, newIndexName); + return resolveAliasRolloverNames(currentState.metadata(), indexAbstraction, newIndexName); case DATA_STREAM: return resolveDataStreamRolloverNames(currentState, (IndexAbstraction.DataStream) indexAbstraction); default: @@ -150,8 +151,8 @@ public static class NameResolution { } } - private NameResolution resolveAliasRolloverNames(IndexAbstraction.Alias alias, String newIndexName) { - final IndexMetadata writeIndex = alias.getWriteIndex(); + private NameResolution resolveAliasRolloverNames(Metadata metadata, IndexAbstraction alias, String newIndexName) { + final IndexMetadata writeIndex = metadata.index(alias.getWriteIndex()); final String sourceProvidedName = writeIndex.getSettings().get(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, writeIndex.getIndex().getName()); final String sourceIndexName = writeIndex.getIndex().getName(); @@ -165,7 +166,7 @@ private NameResolution resolveAliasRolloverNames(IndexAbstraction.Alias alias, S private NameResolution resolveDataStreamRolloverNames(ClusterState currentState, IndexAbstraction.DataStream dataStream) { final Version minNodeVersion = currentState.nodes().getMinNodeVersion(); final DataStream ds = dataStream.getDataStream(); - final IndexMetadata originalWriteIndex = dataStream.getWriteIndex(); + final IndexMetadata originalWriteIndex = currentState.metadata().index(dataStream.getWriteIndex()); final DataStream rolledDataStream = ds.rollover(currentState.getMetadata(), "uuid", minNodeVersion); return new NameResolution(originalWriteIndex.getIndex().getName(), null, rolledDataStream.getWriteIndex().getName()); } @@ -173,12 +174,12 @@ private NameResolution resolveDataStreamRolloverNames(ClusterState currentState, private RolloverResult rolloverAlias(ClusterState currentState, IndexAbstraction.Alias alias, String aliasName, String newIndexName, CreateIndexRequest createIndexRequest, List> metConditions, boolean silent, boolean onlyValidate) throws Exception { - final NameResolution names = resolveAliasRolloverNames(alias, newIndexName); + final NameResolution names = resolveAliasRolloverNames(currentState.metadata(), alias, newIndexName); final String sourceIndexName = names.sourceName; final String rolloverIndexName = names.rolloverName; final String unresolvedName = names.unresolvedName; final Metadata metadata = currentState.metadata(); - final IndexMetadata writeIndex = alias.getWriteIndex(); + final IndexMetadata writeIndex = currentState.metadata().index(alias.getWriteIndex()); final AliasMetadata aliasMetadata = writeIndex.getAliases().get(alias.getName()); final boolean explicitWriteIndex = Boolean.TRUE.equals(aliasMetadata.writeIndex()); final Boolean isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.exists(createIndexRequest.settings()) ? @@ -232,11 +233,11 @@ private RolloverResult rolloverDataStream(ClusterState currentState, IndexAbstra final Version minNodeVersion = currentState.nodes().getMinNodeVersion(); final DataStream ds = dataStream.getDataStream(); - final IndexMetadata originalWriteIndex = dataStream.getWriteIndex(); + final Index originalWriteIndex = dataStream.getWriteIndex(); DataStream rolledDataStream = ds.rollover(currentState.metadata(), "uuid", minNodeVersion); createIndexService.validateIndexName(rolledDataStream.getWriteIndex().getName(), currentState); // fails if the index already exists if (onlyValidate) { - return new RolloverResult(rolledDataStream.getWriteIndex().getName(), originalWriteIndex.getIndex().getName(), currentState); + return new RolloverResult(rolledDataStream.getWriteIndex().getName(), originalWriteIndex.getName(), currentState); } CreateIndexClusterStateUpdateRequest createIndexClusterStateRequest = prepareDataStreamCreateIndexRequest( @@ -251,11 +252,11 @@ private RolloverResult rolloverDataStream(ClusterState currentState, IndexAbstra RolloverInfo rolloverInfo = new RolloverInfo(dataStreamName, metConditions, threadPool.absoluteTimeInMillis()); newState = ClusterState.builder(newState) .metadata(Metadata.builder(newState.metadata()) - .put(IndexMetadata.builder(newState.metadata().index(originalWriteIndex.getIndex())) + .put(IndexMetadata.builder(newState.metadata().index(originalWriteIndex)) .putRolloverInfo(rolloverInfo))) .build(); - return new RolloverResult(rolledDataStream.getWriteIndex().getName(), originalWriteIndex.getIndex().getName(), newState); + return new RolloverResult(rolledDataStream.getWriteIndex().getName(), originalWriteIndex.getName(), newState); } static String generateRolloverIndexName(String sourceIndexName, IndexNameExpressionResolver indexNameExpressionResolver) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index 8f710a13926bb..37478c1fb23a2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -241,12 +241,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public static AliasMetadata getFirstAliasMetadata(IndexAbstraction ia) { + public static AliasMetadata getFirstAliasMetadata(Metadata metadata, IndexAbstraction ia) { if (ia.getType() != IndexAbstraction.Type.ALIAS) { throw new IllegalArgumentException("unexpected type: [" + ia.getType() + "]"); } - return ia.getIndices().get(0).getAliases().get(ia.getName()); + IndexMetadata firstIndex = metadata.index(ia.getIndices().get(0)); + return firstIndex.getAliases().get(ia.getName()); } public static class Builder { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index b4bec136a8f31..89c4253c7c071 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -263,7 +263,7 @@ public DataStream addBackingIndex(Metadata clusterMetadata, Index index) { } // ensure that no aliases reference index - IndexMetadata im = clusterMetadata.getIndicesLookup().get(index.getName()).getWriteIndex(); + IndexMetadata im = clusterMetadata.index(clusterMetadata.getIndicesLookup().get(index.getName()).getWriteIndex()); if (im.getAliases().size() > 0) { throw new IllegalArgumentException( String.format(Locale.ROOT, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index e037c2e134b2e..37048035fe476 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -9,7 +9,9 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; +import org.elasticsearch.index.Index; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -17,9 +19,6 @@ import java.util.Objects; import java.util.stream.Collectors; -import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_HIDDEN_SETTING; -import static org.elasticsearch.core.List.copyOf; - /** * An index abstraction is a reference to one or more concrete indices. * An index abstraction has a unique name and encapsulates all the {@link IndexMetadata} instances it is pointing to. @@ -40,7 +39,7 @@ public interface IndexAbstraction { /** * @return All {@link IndexMetadata} of all concrete indices this index abstraction is referring to. */ - List getIndices(); + List getIndices(); /** * A write index is a dedicated concrete index, that accepts all the new documents that belong to an index abstraction. @@ -52,7 +51,7 @@ public interface IndexAbstraction { * null if this index abstraction doesn't have a write index. */ @Nullable - IndexMetadata getWriteIndex(); + Index getWriteIndex(); /** * @return the data stream to which this index belongs or null if this is not a concrete index or @@ -123,23 +122,30 @@ public String getDisplayName() { /** * Represents an concrete index and encapsulates its {@link IndexMetadata} */ - class Index implements IndexAbstraction { + class ConcreteIndex implements IndexAbstraction { - private final IndexMetadata concreteIndex; + private final Index concreteIndexName; + private final boolean isHidden; + private final boolean isSystem; + private final List aliases; private final DataStream dataStream; - public Index(IndexMetadata indexMetadata, DataStream dataStream) { - this.concreteIndex = indexMetadata; + public ConcreteIndex(IndexMetadata indexMetadata, DataStream dataStream) { + this.concreteIndexName = indexMetadata.getIndex(); + this.isHidden = indexMetadata.isHidden(); + this.isSystem = indexMetadata.isSystem(); + this.aliases = indexMetadata.getAliases() != null ? + Arrays.asList(indexMetadata.getAliases().keys().toArray(String.class)) : null; this.dataStream = dataStream; } - public Index(IndexMetadata indexMetadata) { + public ConcreteIndex(IndexMetadata indexMetadata) { this(indexMetadata, null); } @Override public String getName() { - return concreteIndex.getIndex().getName(); + return concreteIndexName.getName(); } @Override @@ -148,13 +154,13 @@ public Type getType() { } @Override - public List getIndices() { - return Collections.singletonList(concreteIndex); + public List getIndices() { + return Collections.singletonList(concreteIndexName); } @Override - public IndexMetadata getWriteIndex() { - return concreteIndex; + public Index getWriteIndex() { + return concreteIndexName; } @Override @@ -164,17 +170,17 @@ public DataStream getParentDataStream() { @Override public boolean isHidden() { - return INDEX_HIDDEN_SETTING.get(concreteIndex.getSettings()); + return isHidden; } @Override public boolean isSystem() { - return concreteIndex.isSystem(); + return isSystem; } @Override public List getAliases() { - return Arrays.asList(concreteIndex.getAliases().keys().toArray(String.class)); + return aliases; } } @@ -184,28 +190,32 @@ public List getAliases() { class Alias implements IndexAbstraction { private final String aliasName; - private final List referenceIndexMetadatas; - private final IndexMetadata writeIndex; + private final List referenceIndexMetadatas; + private final Index writeIndex; private final boolean isHidden; + private final boolean isSystem; private final boolean dataStreamAlias; public Alias(AliasMetadata aliasMetadata, List indices) { this.aliasName = aliasMetadata.getAlias(); - this.referenceIndexMetadatas = indices; + this.referenceIndexMetadatas = new ArrayList<>(indices.size()); + for (IndexMetadata imd : indices) { + this.referenceIndexMetadatas.add(imd.getIndex()); + } List writeIndices = indices.stream() .filter(idxMeta -> Boolean.TRUE.equals(idxMeta.getAliases().get(aliasName).writeIndex())) .collect(Collectors.toList()); - if (writeIndices.isEmpty() && referenceIndexMetadatas.size() == 1 - && referenceIndexMetadatas.get(0).getAliases().get(aliasName).writeIndex() == null) { - writeIndices.add(referenceIndexMetadatas.get(0)); + if (writeIndices.isEmpty() && indices.size() == 1 + && indices.get(0).getAliases().get(aliasName).writeIndex() == null) { + writeIndices.add(indices.get(0)); } if (writeIndices.size() == 0) { this.writeIndex = null; } else if (writeIndices.size() == 1) { - this.writeIndex = writeIndices.get(0); + this.writeIndex = writeIndices.get(0).getIndex(); } else { List writeIndicesStrings = writeIndices.stream() .map(i -> i.getIndex().getName()).collect(Collectors.toList()); @@ -214,17 +224,17 @@ public Alias(AliasMetadata aliasMetadata, List indices) { } this.isHidden = aliasMetadata.isHidden() == null ? false : aliasMetadata.isHidden(); + this.isSystem = indices.stream().allMatch(IndexMetadata::isSystem); dataStreamAlias = false; - validateAliasProperties(); + validateAliasProperties(indices); } - public Alias(org.elasticsearch.cluster.metadata.DataStreamAlias dataStreamAlias, - List indicesOfAllDataStreams, - IndexMetadata writeIndexOfWriteDataStream) { + public Alias(DataStreamAlias dataStreamAlias, List indicesOfAllDataStreams, Index writeIndexOfWriteDataStream) { this.aliasName = dataStreamAlias.getName(); this.referenceIndexMetadatas = indicesOfAllDataStreams; this.writeIndex = writeIndexOfWriteDataStream; this.isHidden = false; + this.isSystem = false; this.dataStreamAlias = true; } @@ -238,12 +248,12 @@ public String getName() { } @Override - public List getIndices() { + public List getIndices() { return referenceIndexMetadatas; } @Nullable - public IndexMetadata getWriteIndex() { + public Index getWriteIndex() { return writeIndex; } @@ -260,7 +270,7 @@ public boolean isHidden() { @Override public boolean isSystem() { - return referenceIndexMetadatas.stream().allMatch(IndexMetadata::isSystem); + return isSystem; } @Override @@ -273,7 +283,7 @@ public List getAliases() { return null; } - private void validateAliasProperties() { + private void validateAliasProperties(List referenceIndexMetadatas) { // Validate hidden status final Map> groupedByHiddenStatus = referenceIndexMetadatas.stream() .collect(Collectors.groupingBy(idxMeta -> Boolean.TRUE.equals(idxMeta.getAliases().get(aliasName).isHidden()))); @@ -297,16 +307,11 @@ private boolean isNonEmpty(List idxMetas) { class DataStream implements IndexAbstraction { private final org.elasticsearch.cluster.metadata.DataStream dataStream; - private final List dataStreamIndices; - private final IndexMetadata writeIndex; private final List referencedByDataStreamAliases; public DataStream(org.elasticsearch.cluster.metadata.DataStream dataStream, - List dataStreamIndices, List aliases) { this.dataStream = dataStream; - this.dataStreamIndices = copyOf(dataStreamIndices); - this.writeIndex = dataStreamIndices.get(dataStreamIndices.size() - 1); this.referencedByDataStreamAliases = aliases; } @@ -321,12 +326,12 @@ public Type getType() { } @Override - public List getIndices() { - return dataStreamIndices; + public List getIndices() { + return dataStream.getIndices(); } - public IndexMetadata getWriteIndex() { - return writeIndex; + public Index getWriteIndex() { + return dataStream.getWriteIndex(); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java index f29ddef978963..add6ff97597fc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java @@ -204,7 +204,7 @@ public static boolean isIndexVisible(String expression, String index, IndicesOpt } } - IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0); + IndexMetadata indexMetadata = metadata.index(indexAbstraction.getIndices().get(0)); if (indexMetadata.getState() == IndexMetadata.State.CLOSE && indicesOptions.expandWildcardsClosed()) { return true; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 285b8aa5983a8..288e2cdc2adff 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -239,8 +239,9 @@ Index[] concreteIndices(Context context, String... indexExpressions) { boolean excludedDataStreams = false; final Set concreteIndices = new LinkedHashSet<>(expressions.size()); + final SortedMap indicesLookup = context.state.metadata().getIndicesLookup(); for (String expression : expressions) { - IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(expression); + IndexAbstraction indexAbstraction = indicesLookup.get(expression); if (indexAbstraction == null ) { if (failNoIndices) { IndexNotFoundException infe; @@ -266,35 +267,35 @@ Index[] concreteIndices(Context context, String... indexExpressions) { } if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.isResolveToWriteIndex()) { - IndexMetadata writeIndex = indexAbstraction.getWriteIndex(); + Index writeIndex = indexAbstraction.getWriteIndex(); if (writeIndex == null) { throw new IllegalArgumentException("no write index is defined for alias [" + indexAbstraction.getName() + "]." + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + " indices without one being designated as a write index"); } - if (addIndex(writeIndex, context)) { - concreteIndices.add(writeIndex.getIndex()); + if (addIndex(writeIndex, null, context)) { + concreteIndices.add(writeIndex); } } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.isResolveToWriteIndex()) { - IndexMetadata writeIndex = indexAbstraction.getWriteIndex(); - if (addIndex(writeIndex, context)) { - concreteIndices.add(writeIndex.getIndex()); + Index writeIndex = indexAbstraction.getWriteIndex(); + if (addIndex(writeIndex, null, context)) { + concreteIndices.add(writeIndex); } } else { if (indexAbstraction.getIndices().size() > 1 && options.allowAliasesToMultipleIndices() == false) { String[] indexNames = new String[indexAbstraction.getIndices().size()]; int i = 0; - for (IndexMetadata indexMetadata : indexAbstraction.getIndices()) { - indexNames[i++] = indexMetadata.getIndex().getName(); + for (Index indexName : indexAbstraction.getIndices()) { + indexNames[i++] = indexName.getName(); } throw new IllegalArgumentException(indexAbstraction.getType().getDisplayName() + " [" + expression + "] has more than one index associated with it " + Arrays.toString(indexNames) + ", can't execute a single index op"); } - for (IndexMetadata index : indexAbstraction.getIndices()) { + for (Index index : indexAbstraction.getIndices()) { if (shouldTrackConcreteIndex(context, options, index)) { - concreteIndices.add(index.getIndex()); + concreteIndices.add(index); } } } @@ -309,11 +310,12 @@ Index[] concreteIndices(Context context, String... indexExpressions) { } throw infe; } - checkSystemIndexAccess(context, metadata, concreteIndices, indexExpressions); + checkSystemIndexAccess(context, concreteIndices, indexExpressions); return concreteIndices.toArray(Index.EMPTY_ARRAY); } - private void checkSystemIndexAccess(Context context, Metadata metadata, Set concreteIndices, String[] originalPatterns) { + private void checkSystemIndexAccess(Context context, Set concreteIndices, String[] originalPatterns) { + final Metadata metadata = context.getState().metadata(); final Predicate systemIndexAccessPredicate = context.getSystemIndexAccessPredicate().negate(); final List systemIndicesThatShouldNotBeAccessed = concreteIndices.stream() .map(metadata::index) @@ -354,32 +356,38 @@ private void checkSystemIndexAccess(Context context, Metadata metadata, Set> resolveSearchRouting(ClusterState state, @Nullab for (String expression : resolvedExpressions) { IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(expression); if (indexAbstraction != null && indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) { - for (IndexMetadata index : indexAbstraction.getIndices()) { - String concreteIndex = index.getIndex().getName(); - AliasMetadata aliasMetadata = index.getAliases().get(indexAbstraction.getName()); + for (Index index : indexAbstraction.getIndices()) { + String concreteIndex = index.getName(); if (norouting.contains(concreteIndex) == false) { + AliasMetadata aliasMetadata = state.metadata().index(concreteIndex).getAliases().get(indexAbstraction.getName()); if (aliasMetadata != null && aliasMetadata.searchRoutingValues().isEmpty() == false) { // Routing alias if (routings == null) { @@ -1126,7 +1134,8 @@ private static Set expand(Context context, IndexMetadata.State excludeSt if (context.isPreserveAliases() && indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) { expand.add(aliasOrIndexName); } else { - for (IndexMetadata meta : indexAbstraction.getIndices()) { + for (Index index : indexAbstraction.getIndices()) { + IndexMetadata meta = context.state.metadata().index(index); if (excludeState == null || meta.getState() != excludeState) { expand.add(meta.getIndex().getName()); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 7fba6699ca76d..d2c1c4bf6bf79 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.coordination.CoordinationMetadata; +import org.elasticsearch.cluster.metadata.IndexAbstraction.ConcreteIndex; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.HppcMaps; @@ -635,11 +636,11 @@ public String resolveWriteIndexRouting(@Nullable String routing, String aliasOrI if (result == null || result.getType() != IndexAbstraction.Type.ALIAS) { return routing; } - IndexMetadata writeIndex = result.getWriteIndex(); - if (writeIndex == null) { + Index writeIndexName = result.getWriteIndex(); + if (writeIndexName == null) { throw new IllegalArgumentException("alias [" + aliasOrIndex + "] does not have a write index"); } - AliasMetadata aliasMd = writeIndex.getAliases().get(result.getName()); + AliasMetadata aliasMd = index(writeIndexName).getAliases().get(result.getName()); if (aliasMd != null && aliasMd.indexRouting() != null) { if (aliasMd.indexRouting().indexOf(',') != -1) { throw new IllegalArgumentException("index/alias [" + aliasOrIndex + "] provided with routing value [" @@ -674,7 +675,7 @@ public String resolveIndexRouting(@Nullable String routing, String aliasOrIndex) if (result.getIndices().size() > 1) { rejectSingleIndexOperation(aliasOrIndex, result); } - AliasMetadata aliasMd = AliasMetadata.getFirstAliasMetadata(result); + AliasMetadata aliasMd = AliasMetadata.getFirstAliasMetadata(this, result); if (aliasMd.indexRouting() != null) { if (aliasMd.indexRouting().indexOf(',') != -1) { throw new IllegalArgumentException("index/alias [" + aliasOrIndex + "] provided with routing value [" + @@ -695,8 +696,8 @@ public String resolveIndexRouting(@Nullable String routing, String aliasOrIndex) private void rejectSingleIndexOperation(String aliasOrIndex, IndexAbstraction result) { String[] indexNames = new String[result.getIndices().size()]; int i = 0; - for (IndexMetadata indexMetadata : result.getIndices()) { - indexNames[i++] = indexMetadata.getIndex().getName(); + for (Index indexName : result.getIndices()) { + indexNames[i++] = indexName.getName(); } throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has more than one index associated with it [" + Arrays.toString(indexNames) + "], can't execute a single index op"); @@ -1695,34 +1696,29 @@ static SortedMap buildIndicesLookup(DataStreamMetadata if (dataStreamMetadata != null && indices.size() > 0) { Map> dataStreamToAliasLookup = new HashMap<>(); for (DataStreamAlias alias : dataStreamMetadata.getDataStreamAliases().values()) { - List allIndicesOfAllDataStreams = alias.getDataStreams().stream() + List allIndicesOfAllDataStreams = alias.getDataStreams().stream() .map(name -> { List aliases = dataStreamToAliasLookup.computeIfAbsent(name, k -> new LinkedList<>()); aliases.add(alias.getName()); return dataStreamMetadata.dataStreams().get(name); }) .flatMap(ds -> ds.getIndices().stream()) - .map(index -> indices.get(index.getName())) .collect(Collectors.toList()); - IndexMetadata writeIndexOfWriteDataStream = null; + Index writeIndexOfWriteDataStream = null; if (alias.getWriteDataStream() != null) { DataStream writeDataStream = dataStreamMetadata.dataStreams().get(alias.getWriteDataStream()); - writeIndexOfWriteDataStream = indices.get(writeDataStream.getWriteIndex().getName()); + writeIndexOfWriteDataStream = writeDataStream.getWriteIndex(); } IndexAbstraction existing = indicesLookup.put(alias.getName(), new IndexAbstraction.Alias(alias, allIndicesOfAllDataStreams, writeIndexOfWriteDataStream)); assert existing == null : "duplicate data stream alias for " + alias.getName(); } for (DataStream dataStream : dataStreamMetadata.dataStreams().values()) { - List backingIndices = dataStream.getIndices().stream() - .map(index -> indices.get(index.getName())) - .collect(Collectors.toList()); - assert backingIndices.isEmpty() == false; - assert backingIndices.contains(null) == false; + assert dataStream.getIndices().isEmpty() == false; List aliases = dataStreamToAliasLookup.getOrDefault(dataStream.getName(), Collections.emptyList()); IndexAbstraction existing = indicesLookup.put(dataStream.getName(), - new IndexAbstraction.DataStream(dataStream, backingIndices, aliases)); + new IndexAbstraction.DataStream(dataStream, aliases)); assert existing == null : "duplicate data stream for " + dataStream.getName(); for (Index i : dataStream.getIndices()) { @@ -1733,7 +1729,7 @@ static SortedMap buildIndicesLookup(DataStreamMetadata Map> aliasToIndices = new HashMap<>(); for (IndexMetadata indexMetadata : indices.values()) { - IndexAbstraction.Index index; + ConcreteIndex index; DataStream parent = indexToDataStreamLookup.get(indexMetadata.getIndex().getName()); if (parent != null) { assert parent.getIndices().stream() @@ -1741,9 +1737,9 @@ static SortedMap buildIndicesLookup(DataStreamMetadata .collect(Collectors.toList()) .contains(indexMetadata.getIndex().getName()) : "Expected data stream [" + parent.getName() + "] to contain index " + indexMetadata.getIndex(); - index = new IndexAbstraction.Index(indexMetadata, (IndexAbstraction.DataStream) indicesLookup.get(parent.getName())); + index = new ConcreteIndex(indexMetadata, (IndexAbstraction.DataStream) indicesLookup.get(parent.getName())); } else { - index = new IndexAbstraction.Index(indexMetadata); + index = new ConcreteIndex(indexMetadata); } IndexAbstraction existing = indicesLookup.put(indexMetadata.getIndex().getName(), index); @@ -1773,8 +1769,8 @@ static void validateDataStreams(SortedMap indicesLooku .filter(ia -> ia.getType() == IndexAbstraction.Type.ALIAS) .filter(ia -> ia.isDataStreamRelated() == false) .filter(ia -> { - for (IndexMetadata index : ia.getIndices()) { - if (indicesLookup.get(index.getIndex().getName()).getParentDataStream() != null) { + for (Index index : ia.getIndices()) { + if (indicesLookup.get(index.getName()).getParentDataStream() != null) { return true; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 540d878845c16..559ee94db02e1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1234,7 +1234,7 @@ static IndexMetadata validateResize(ClusterState state, String sourceIndex, Stri IndexAbstraction source = state.metadata().getIndicesLookup().get(sourceIndex); assert source != null; if (source.getParentDataStream() != null && - source.getParentDataStream().getWriteIndex().getIndex().equals(sourceMetadata.getIndex())) { + source.getParentDataStream().getWriteIndex().equals(sourceMetadata.getIndex())) { throw new IllegalArgumentException(String.format(Locale.ROOT, "cannot resize the write index [%s] for data stream [%s]", sourceIndex, source.getParentDataStream().getName())); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java index 1fd597cc7f9dc..8f16b39d7b51f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java @@ -80,7 +80,7 @@ public ClusterState deleteIndices(ClusterState currentState, Set indices) IndexMetadata im = meta.getIndexSafe(index); IndexAbstraction.DataStream parent = meta.getIndicesLookup().get(im.getIndex().getName()).getParentDataStream(); if (parent != null) { - if (parent.getWriteIndex().equals(im)) { + if (parent.getWriteIndex().equals(im.getIndex())) { throw new IllegalArgumentException("index [" + index.getName() + "] is the write index for data stream [" + parent.getName() + "] and cannot be deleted"); } else { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java index 470c5b0bbd6fd..bcdb54ef8e21f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java @@ -369,12 +369,16 @@ public void addIndexBlock(AddIndexBlockClusterStateUpdateRequest request, if (concreteIndices == null || concreteIndices.length == 0) { throw new IllegalArgumentException("Index name is required"); } + Metadata metadata = clusterService.state().metadata(); List writeIndices = new ArrayList<>(); - SortedMap lookup = clusterService.state().metadata().getIndicesLookup(); + SortedMap lookup = metadata.getIndicesLookup(); for (Index index : concreteIndices) { IndexAbstraction ia = lookup.get(index.getName()); - if (ia != null && ia.getParentDataStream() != null && ia.getParentDataStream().getWriteIndex().getIndex().equals(index)) { - writeIndices.add(index.getName()); + if (ia != null && ia.getParentDataStream() != null) { + Index writeIndex = metadata.index(ia.getParentDataStream().getWriteIndex()).getIndex(); + if (writeIndex.equals(index)) { + writeIndices.add(index.getName()); + } } } if (writeIndices.size() > 0) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 2e3ce2f827a65..73136860d9474 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; @@ -122,21 +123,24 @@ static ClusterState migrateToDataStream(ClusterState currentState, validateBackingIndices(currentState, request.aliasName); Metadata.Builder mb = Metadata.builder(currentState.metadata()); - for (IndexMetadata im : alias.getIndices()) { + for (Index index : alias.getIndices()) { + IndexMetadata im = currentState.metadata().index(index); prepareBackingIndex(mb, im, request.aliasName, mapperSupplier); } currentState = ClusterState.builder(currentState).metadata(mb).build(); - IndexMetadata writeIndex = alias.getWriteIndex(); + Index writeIndex = alias.getWriteIndex(); + ClusterState finalCurrentState = currentState; List backingIndices = alias.getIndices() .stream() - .filter(x -> writeIndex == null || x.getIndex().getName().equals(writeIndex.getIndex().getName()) == false) + .filter(x -> writeIndex == null || x.equals(writeIndex) == false) + .map(x -> finalCurrentState.metadata().index(x)) .collect(Collectors.toList()); logger.info("submitting request to migrate alias [{}] to a data stream", request.aliasName); CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(request.aliasName); - return createDataStream(metadataCreateIndexService, currentState, req, backingIndices, writeIndex); + return createDataStream(metadataCreateIndexService, currentState, req, backingIndices, currentState.metadata().index(writeIndex)); } // package-visible for testing @@ -150,7 +154,7 @@ static void validateRequest(ClusterState currentState, MigrateToDataStreamCluste } // check for "clean" alias without routing or filter query - AliasMetadata aliasMetadata = AliasMetadata.getFirstAliasMetadata(ia); + AliasMetadata aliasMetadata = AliasMetadata.getFirstAliasMetadata(currentState.metadata(), ia); assert aliasMetadata != null : "alias metadata may not be null"; if (aliasMetadata.filteringRequired() || aliasMetadata.getIndexRouting() != null || aliasMetadata.getSearchRouting() != null) { throw new IllegalArgumentException("alias [" + request.aliasName + "] may not have custom filtering or routing"); @@ -192,9 +196,10 @@ static void validateBackingIndices(ClusterState currentState, String dataStreamN // ensure that no other aliases reference indices List indicesWithOtherAliases = new ArrayList<>(); - for (IndexMetadata im : alias.getIndices()) { + for (Index index : alias.getIndices()) { + IndexMetadata im = currentState.metadata().index(index); if (im.getAliases().size() > 1 || im.getAliases().containsKey(alias.getName()) == false) { - indicesWithOtherAliases.add(im.getIndex().getName()); + indicesWithOtherAliases.add(index.getName()); } } if (indicesWithOtherAliases.size() > 0) { diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index bbd51ec96f869..692aab89b115e 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -149,15 +149,15 @@ public static boolean resolvePipelines(final DocWriteRequest originalRequest, // check the alias for the index request (this is how normal index requests are modeled) if (indexMetadata == null && indexRequest.index() != null) { IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(indexRequest.index()); - if (indexAbstraction != null) { - indexMetadata = indexAbstraction.getWriteIndex(); + if (indexAbstraction != null && indexAbstraction.getWriteIndex() != null) { + indexMetadata = metadata.index(indexAbstraction.getWriteIndex()); } } // check the alias for the action request (this is how upserts are modeled) if (indexMetadata == null && originalRequest != null && originalRequest.index() != null) { IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(originalRequest.index()); - if (indexAbstraction != null) { - indexMetadata = indexAbstraction.getWriteIndex(); + if (indexAbstraction != null && indexAbstraction.getWriteIndex() != null) { + indexMetadata = metadata.index(indexAbstraction.getWriteIndex()); } } if (indexMetadata != null) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index 46f4070b2353b..e359a2f916983 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -161,7 +161,7 @@ public void testResolveIndicesWithWriteIndexOnlyAndDataStreamsAndWriteAliases() List indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList()); IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo"); // should resolve the data stream and each alias to their respective write indices - assertThat(indexNames, containsInAnyOrder(expectedDs.getWriteIndex().getIndex().getName(), "index2", "index3")); + assertThat(indexNames, containsInAnyOrder(expectedDs.getWriteIndex().getName(), "index2", "index3")); } public void testResolveIndicesWithoutWriteIndexOnlyAndDataStreamsAndWriteAliases() { @@ -181,7 +181,7 @@ public void testResolveIndicesWithoutWriteIndexOnlyAndDataStreamsAndWriteAliases Index[] indices = TransportPutMappingAction.resolveIndices(cs, request, TestIndexNameExpressionResolver.newInstance()); List indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList()); IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo"); - List expectedIndices = expectedDs.getIndices().stream().map(im -> im.getIndex().getName()).collect(Collectors.toList()); + List expectedIndices = expectedDs.getIndices().stream().map(Index::getName).collect(Collectors.toList()); expectedIndices.addAll(org.elasticsearch.core.List.of("index1", "index2", "index3")); // should resolve the data stream and each alias to _all_ their respective indices assertThat(indexNames, containsInAnyOrder(expectedIndices.toArray())); @@ -204,10 +204,10 @@ public void testResolveIndicesWithWriteIndexOnlyAndDataStreamAndIndex() { Index[] indices = TransportPutMappingAction.resolveIndices(cs, request, TestIndexNameExpressionResolver.newInstance()); List indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList()); IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo"); - List expectedIndices = expectedDs.getIndices().stream().map(im -> im.getIndex().getName()).collect(Collectors.toList()); + List expectedIndices = expectedDs.getIndices().stream().map(Index::getName).collect(Collectors.toList()); expectedIndices.addAll(org.elasticsearch.core.List.of("index1", "index2", "index3")); // should resolve the data stream and each alias to _all_ their respective indices - assertThat(indexNames, containsInAnyOrder(expectedDs.getWriteIndex().getIndex().getName(), "index3")); + assertThat(indexNames, containsInAnyOrder(expectedDs.getWriteIndex().getName(), "index3")); } public void testResolveIndicesWithWriteIndexOnlyAndNoSingleWriteIndex() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 6947b530aa2ea..177324bab6c54 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -528,9 +528,9 @@ public void testRolloverClusterState() throws Exception { IndexAbstraction alias = rolloverMetadata.getIndicesLookup().get(aliasName); assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS)); assertThat(alias.getIndices(), hasSize(2)); - assertThat(alias.getIndices(), hasItem(rolloverMetadata.index(sourceIndexName))); - assertThat(alias.getIndices(), hasItem(rolloverIndexMetadata)); - assertThat(alias.getWriteIndex(), equalTo(rolloverIndexMetadata)); + assertThat(alias.getIndices(), hasItem(rolloverMetadata.index(sourceIndexName).getIndex())); + assertThat(alias.getIndices(), hasItem(rolloverIndexMetadata.getIndex())); + assertThat(alias.getWriteIndex(), equalTo(rolloverIndexMetadata.getIndex())); RolloverInfo info = rolloverMetadata.index(sourceIndexName).getRolloverInfos().get(aliasName); assertThat(info.getTime(), lessThanOrEqualTo(after)); @@ -618,9 +618,9 @@ public void testRolloverClusterStateForDataStream() throws Exception { IndexAbstraction ds = rolloverMetadata.getIndicesLookup().get(dataStream.getName()); assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); assertThat(ds.getIndices(), hasSize(dataStream.getIndices().size() + 1)); - assertThat(ds.getIndices(), hasItem(rolloverMetadata.index(sourceIndexName))); - assertThat(ds.getIndices(), hasItem(rolloverIndexMetadata)); - assertThat(ds.getWriteIndex(), equalTo(rolloverIndexMetadata)); + assertThat(ds.getIndices(), hasItem(rolloverMetadata.index(sourceIndexName).getIndex())); + assertThat(ds.getIndices(), hasItem(rolloverIndexMetadata.getIndex())); + assertThat(ds.getWriteIndex(), equalTo(rolloverIndexMetadata.getIndex())); RolloverInfo info = rolloverMetadata.index(sourceIndexName).getRolloverInfos().get(dataStream.getName()); assertThat(info.getTime(), lessThanOrEqualTo(after)); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 2951ee0b45b2c..c9604c47b5f19 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexAbstraction.Index; +import org.elasticsearch.cluster.metadata.IndexAbstraction.ConcreteIndex; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -243,9 +243,9 @@ public void testOnlySystem() { SortedMap indicesLookup = new TreeMap<>(); Settings settings = Settings.builder().put("index.version.created", Version.CURRENT).build(); indicesLookup.put(".foo", - new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); + new ConcreteIndex(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); indicesLookup.put(".bar", - new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); + new ConcreteIndex(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); SystemIndices systemIndices = new SystemIndices( Map.of("plugin", new SystemIndices.Feature( "plugin", diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java index ad663ad0f71ee..fc2c96f773b5c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java @@ -76,7 +76,7 @@ public void testAddAndRemove() { IndexAbstraction alias = after.metadata().getIndicesLookup().get("test"); assertNotNull(alias); assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS)); - assertThat(alias.getIndices(), contains(after.metadata().index(index))); + assertThat(alias.getIndices(), contains(after.metadata().index(index).getIndex())); assertAliasesVersionIncreased(index, before, after); // Remove the alias from it while adding another one @@ -88,7 +88,7 @@ public void testAddAndRemove() { alias = after.metadata().getIndicesLookup().get("test_2"); assertNotNull(alias); assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS)); - assertThat(alias.getIndices(), contains(after.metadata().index(index))); + assertThat(alias.getIndices(), contains(after.metadata().index(index).getIndex())); assertAliasesVersionIncreased(index, before, after); // Now just remove on its own @@ -110,7 +110,7 @@ public void testMustExist() { IndexAbstraction alias = after.metadata().getIndicesLookup().get("test"); assertNotNull(alias); assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS)); - assertThat(alias.getIndices(), contains(after.metadata().index(index))); + assertThat(alias.getIndices(), contains(after.metadata().index(index).getIndex())); assertAliasesVersionIncreased(index, before, after); // Remove the alias from it with mustExist == true while adding another one @@ -122,7 +122,7 @@ public void testMustExist() { alias = after.metadata().getIndicesLookup().get("test_2"); assertNotNull(alias); assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS)); - assertThat(alias.getIndices(), contains(after.metadata().index(index))); + assertThat(alias.getIndices(), contains(after.metadata().index(index).getIndex())); assertAliasesVersionIncreased(index, before, after); // Now just remove on its own @@ -240,7 +240,7 @@ public void testSwapIndexWithAlias() { IndexAbstraction alias = after.metadata().getIndicesLookup().get("test"); assertNotNull(alias); assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS)); - assertThat(alias.getIndices(), contains(after.metadata().index("test_2"))); + assertThat(alias.getIndices(), contains(after.metadata().index("test_2").getIndex())); assertAliasesVersionIncreased("test_2", before, after); } @@ -278,14 +278,14 @@ public void testAddWriteOnlyWithNoExistingAliases() { after = service.applyAliasActions(before, Arrays.asList( new AliasAction.Add("test", "alias", null, null, null, null, null))); assertNull(after.metadata().index("test").getAliases().get("alias").writeIndex()); - assertThat(after.metadata().getIndicesLookup().get("alias").getWriteIndex(), + assertThat(after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()), equalTo(after.metadata().index("test"))); assertAliasesVersionIncreased("test", before, after); after = service.applyAliasActions(before, Arrays.asList( new AliasAction.Add("test", "alias", null, null, null, true, null))); assertTrue(after.metadata().index("test").getAliases().get("alias").writeIndex()); - assertThat(after.metadata().getIndicesLookup().get("alias").getWriteIndex(), + assertThat(after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()), equalTo(after.metadata().index("test"))); assertAliasesVersionIncreased("test", before, after); } @@ -302,7 +302,7 @@ public void testAddWriteOnlyWithExistingWriteIndex() { ClusterState after = service.applyAliasActions(before, Arrays.asList( new AliasAction.Add("test", "alias", null, null, null, null, null))); assertNull(after.metadata().index("test").getAliases().get("alias").writeIndex()); - assertThat(after.metadata().getIndicesLookup().get("alias").getWriteIndex(), + assertThat(after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()), equalTo(after.metadata().index("test2"))); assertAliasesVersionIncreased("test", before, after); assertAliasesVersionUnchanged("test2", before, after); @@ -330,7 +330,7 @@ public void testSwapWriteOnlyIndex() { ClusterState after = service.applyAliasActions(before, swapActions); assertThat(after.metadata().index("test").getAliases().get("alias").writeIndex(), equalTo(unsetValue)); assertTrue(after.metadata().index("test2").getAliases().get("alias").writeIndex()); - assertThat(after.metadata().getIndicesLookup().get("alias").getWriteIndex(), + assertThat(after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()), equalTo(after.metadata().index("test2"))); assertAliasesVersionIncreased("test", before, after); assertAliasesVersionIncreased("test2", before, after); @@ -353,7 +353,7 @@ public void testAddWriteOnlyWithExistingNonWriteIndices() { ClusterState after = service.applyAliasActions(before, Arrays.asList( new AliasAction.Add("test3", "alias", null, null, null, true, null))); assertTrue(after.metadata().index("test3").getAliases().get("alias").writeIndex()); - assertThat(after.metadata().getIndicesLookup().get("alias").getWriteIndex(), + assertThat(after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()), equalTo(after.metadata().index("test3"))); assertAliasesVersionUnchanged("test", before, after); assertAliasesVersionUnchanged("test2", before, after); @@ -376,7 +376,7 @@ public void testAddWriteOnlyWithIndexRemoved() { ClusterState after = service.applyAliasActions(before, singletonList(new AliasAction.RemoveIndex("test"))); assertNull(after.metadata().index("test2").getAliases().get("alias").writeIndex()); - assertThat(after.metadata().getIndicesLookup().get("alias").getWriteIndex(), + assertThat(after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()), equalTo(after.metadata().index("test2"))); assertAliasesVersionUnchanged("test2", before, after); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index 419a1ae48f840..9255ae401ec8a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.indices.EmptySystemIndices; @@ -220,10 +221,11 @@ public void testCreateDataStreamWithSuppliedWriteIndex() throws Exception { assertThat(ds, notNullValue()); assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); assertThat(ds.getIndices().size(), equalTo(2)); - List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); + List backingIndexNames = ds.getIndices().stream().map(Index::getName).collect(Collectors.toList()); assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2")); - assertThat(ds.getWriteIndex().getIndex().getName(), equalTo("foo1")); - for (IndexMetadata im : ds.getIndices()) { + assertThat(ds.getWriteIndex().getName(), equalTo("foo1")); + for (Index index : ds.getIndices()) { + IndexMetadata im = newState.metadata().index(index); assertThat(im.getSettings().get("index.hidden"), equalTo("true")); assertThat(im.getAliases().size(), equalTo(0)); } @@ -265,10 +267,11 @@ public void testCreateDataStreamHidesBackingIndicesAndRemovesAlias() throws Exce assertThat(ds, notNullValue()); assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); assertThat(ds.getIndices().size(), equalTo(2)); - List backingIndexNames = ds.getIndices().stream().map(x -> x.getIndex().getName()).collect(Collectors.toList()); + List backingIndexNames = ds.getIndices().stream().map(Index::getName).collect(Collectors.toList()); assertThat(backingIndexNames, containsInAnyOrder("foo1", "foo2")); - assertThat(ds.getWriteIndex().getIndex().getName(), equalTo("foo1")); - for (IndexMetadata im : ds.getIndices()) { + assertThat(ds.getWriteIndex().getName(), equalTo("foo1")); + for (Index index : ds.getIndices()) { + IndexMetadata im = newState.metadata().index(index); assertThat(im.getSettings().get("index.hidden"), equalTo("true")); assertThat(im.getAliases().size(), equalTo(0)); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 036f3d5ecd26f..ab6b106a4dc5b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -1106,7 +1107,7 @@ public void testBuildIndicesLookupForDataStreams() { assertThat(value.isHidden(), is(false)); assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); assertThat(value.getIndices().size(), equalTo(ds.getIndices().size())); - assertThat(value.getWriteIndex().getIndex().getName(), + assertThat(value.getWriteIndex().getName(), equalTo(DataStream.getDefaultBackingIndexName(name, ds.getGeneration()))); } } @@ -1274,18 +1275,18 @@ public void testValidateDataStreamsAllowsPrefixedBackingIndices() { backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) ); - IndexAbstraction.DataStream dataStreamAbstraction = new IndexAbstraction.DataStream(dataStream, backingIndices, Arrays.asList()); + IndexAbstraction.DataStream dataStreamAbstraction = new IndexAbstraction.DataStream(dataStream, Collections.emptyList()); // manually building the indices lookup as going through Metadata.Builder#build would trigger the validate method already SortedMap indicesLookup = new TreeMap<>(); for (IndexMetadata indexMeta : backingIndices) { - indicesLookup.put(indexMeta.getIndex().getName(), new IndexAbstraction.Index(indexMeta, dataStreamAbstraction)); + indicesLookup.put(indexMeta.getIndex().getName(), new IndexAbstraction.ConcreteIndex(indexMeta, dataStreamAbstraction)); } for (int i = 1; i <= generations; i++) { // for the indices that we added in the data stream with a "shrink-" prefix, add the non-prefixed indices to the lookup if (i % 2 == 0 && i < generations) { IndexMetadata indexMeta = createBackingIndex(dataStreamName, i).build(); - indicesLookup.put(indexMeta.getIndex().getName(), new IndexAbstraction.Index(indexMeta, dataStreamAbstraction)); + indicesLookup.put(indexMeta.getIndex().getName(), new IndexAbstraction.ConcreteIndex(indexMeta, dataStreamAbstraction)); } } DataStreamMetadata dataStreamMetadata = diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index ecdd61884c336..5f8ae6acc5db2 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import org.elasticsearch.xcontent.XContentBuilder; @@ -496,7 +497,7 @@ public AllocationState forecast(long forecastWindow, long now) { .stream() .map(state.metadata().getIndicesLookup()::get) .map(IndexAbstraction.DataStream.class::cast) - .map(ds -> forecast(ds, forecastWindow, now)) + .map(ds -> forecast(state.metadata(), ds, forecastWindow, now)) .filter(Objects::nonNull) .collect(Collectors.toList()); if (singleForecasts.isEmpty()) { @@ -524,15 +525,15 @@ public AllocationState forecast(long forecastWindow, long now) { ); } - private SingleForecast forecast(IndexAbstraction.DataStream stream, long forecastWindow, long now) { - List indices = stream.getIndices(); - if (dataStreamAllocatedToNodes(indices) == false) return null; + private SingleForecast forecast(Metadata metadata, IndexAbstraction.DataStream stream, long forecastWindow, long now) { + List indices = stream.getIndices(); + if (dataStreamAllocatedToNodes(metadata, indices) == false) return null; long minCreationDate = Long.MAX_VALUE; long totalSize = 0; int count = 0; while (count < indices.size()) { ++count; - IndexMetadata indexMetadata = indices.get(indices.size() - count); + IndexMetadata indexMetadata = metadata.index(indices.get(indices.size() - count)); long creationDate = indexMetadata.getCreationDate(); if (creationDate < 0) { return null; @@ -575,7 +576,7 @@ private SingleForecast forecast(IndexAbstraction.DataStream stream, long forecas scaledTotalSize = totalSize; } - IndexMetadata writeIndex = stream.getWriteIndex(); + IndexMetadata writeIndex = metadata.index(stream.getWriteIndex()); Map newIndices = new HashMap<>(); DataStream dataStream = stream.getDataStream(); @@ -606,9 +607,9 @@ private SingleForecast forecast(IndexAbstraction.DataStream stream, long forecas * @param indices the indices of the data stream, in original order from data stream meta. * @return true if the first allocated index is allocated only to the set of nodes. */ - private boolean dataStreamAllocatedToNodes(List indices) { + private boolean dataStreamAllocatedToNodes(Metadata metadata, List indices) { for (int i = 0; i < indices.size(); ++i) { - IndexMetadata indexMetadata = indices.get(indices.size() - i - 1); + IndexMetadata indexMetadata = metadata.index(indices.get(indices.size() - i - 1)); Set inNodes = state.getRoutingTable() .allShards(indexMetadata.getIndex().getName()) .stream() diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java index 697f062eff4e9..8fcbc30213827 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java @@ -385,10 +385,11 @@ private ClusterState.Builder applyCreatedDates( long decrement ) { Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()); - List indices = ds.getIndices(); + List indices = ds.getIndices(); long start = last - (decrement * (indices.size() - 1)); for (int i = 0; i < indices.size(); ++i) { - metadataBuilder.put(IndexMetadata.builder(indices.get(i)).creationDate(start + (i * decrement)).build(), false); + IndexMetadata previousInstance = state.metadata().index(indices.get(i)); + metadataBuilder.put(IndexMetadata.builder(previousInstance).creationDate(start + (i * decrement)).build(), false); } return builder.metadata(metadataBuilder); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java index 81e0a6d1393a8..54150d327134d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java @@ -62,7 +62,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) { IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream(); if (dataStream != null) { assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index"; - if (dataStream.getWriteIndex().getIndex().equals(index)) { + if (dataStream.getWriteIndex().equals(index)) { String errorMessage = String.format(Locale.ROOT, "index [%s] is the write index for data stream [%s], pausing " + "ILM execution of lifecycle [%s] until this index is no longer the write index for the data stream via manual or " + "automated rollover", indexName, dataStream.getName(), policyName); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java index 9092107df0811..93ea45b57bffe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java @@ -40,14 +40,14 @@ public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState cu if (dataStream != null) { assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index"; - if (dataStream.getIndices().size() == 1 && dataStream.getIndices().get(0).equals(indexMetadata)) { + if (dataStream.getIndices().size() == 1 && dataStream.getIndices().get(0).equals(indexMetadata.getIndex())) { // This is the last index in the data stream, the entire stream // needs to be deleted, because we can't have an empty data stream DeleteDataStreamAction.Request deleteReq = new DeleteDataStreamAction.Request(new String[]{dataStream.getName()}); getClient().execute(DeleteDataStreamAction.INSTANCE, deleteReq, ActionListener.wrap(response -> listener.onResponse(null), listener::onFailure)); return; - } else if (dataStream.getWriteIndex().getIndex().getName().equals(indexName)) { + } else if (dataStream.getWriteIndex().getName().equals(indexName)) { String errorMessage = String.format(Locale.ROOT, "index [%s] is the write index for data stream [%s]. " + "stopping execution of lifecycle [%s] as a data stream's write index cannot be deleted. manually rolling over the" + " index will resume the execution of the policy as the index will not be the data stream's write index anymore", diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java index 89e49038b1dd7..30bc2d8cf053e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java @@ -79,7 +79,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) { } assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index"; - if (dataStream.getWriteIndex().getIndex().equals(index)) { + if (dataStream.getWriteIndex().equals(index)) { String errorMessage = String.format(Locale.ROOT, "index [%s] is the write index for data stream [%s], pausing " + "ILM execution of lifecycle [%s] until this index is no longer the write index for the data stream via manual or " + "automated rollover", originalIndex, dataStream.getName(), policyName); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java index 7d182f748aaa2..85f984385ad8c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java @@ -55,7 +55,7 @@ public void performAction(IndexMetadata indexMetadata, ClusterState currentClust IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream(); if (dataStream != null) { assert dataStream.getWriteIndex() != null : "datastream " + dataStream.getName() + " has no write index"; - if (dataStream.getWriteIndex().getIndex().equals(indexMetadata.getIndex()) == false) { + if (dataStream.getWriteIndex().equals(indexMetadata.getIndex()) == false) { logger.warn("index [{}] is not the write index for data stream [{}]. skipping rollover for policy [{}]", indexName, dataStream.getName(), LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings())); listener.onResponse(null); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java index ddb748b40f04d..ce6f16e9d6928 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java @@ -77,10 +77,10 @@ public Result isConditionMet(Index index, ClusterState clusterState) { DataStream dataStream = indexAbstraction.getParentDataStream().getDataStream(); IndexAbstraction dataStreamAbstraction = metadata.getIndicesLookup().get(dataStream.getName()); assert dataStreamAbstraction != null : dataStream.getName() + " datastream is not present in the metadata indices lookup"; - IndexMetadata rolledIndexMeta = dataStreamAbstraction.getWriteIndex(); - if (rolledIndexMeta == null) { + if (dataStreamAbstraction.getWriteIndex() == null) { return getErrorResultOnNullMetadata(getKey(), index); } + IndexMetadata rolledIndexMeta = metadata.index(dataStreamAbstraction.getWriteIndex() ); rolledIndexName = rolledIndexMeta.getIndex().getName(); waitForActiveShardsSettingValue = rolledIndexMeta.getSettings().get(IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey()); } else { @@ -93,26 +93,27 @@ public Result isConditionMet(Index index, ClusterState clusterState) { IndexAbstraction aliasAbstraction = metadata.getIndicesLookup().get(rolloverAlias); assert aliasAbstraction.getType() == IndexAbstraction.Type.ALIAS : rolloverAlias + " must be an alias but it is not"; - IndexMetadata aliasWriteIndex = aliasAbstraction.getWriteIndex(); + Index aliasWriteIndex = aliasAbstraction.getWriteIndex(); if (aliasWriteIndex != null) { - rolledIndexName = aliasWriteIndex.getIndex().getName(); - waitForActiveShardsSettingValue = aliasWriteIndex.getSettings().get(IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey()); + IndexMetadata writeIndexImd = metadata.index(aliasWriteIndex); + rolledIndexName = writeIndexImd.getIndex().getName(); + waitForActiveShardsSettingValue = writeIndexImd.getSettings().get(IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey()); } else { - List indices = aliasAbstraction.getIndices(); + List indices = aliasAbstraction.getIndices(); int maxIndexCounter = -1; - IndexMetadata rolledIndexMeta = null; - for (IndexMetadata indexMetadata : indices) { - int indexNameCounter = parseIndexNameCounter(indexMetadata.getIndex().getName()); + Index tmpRolledIndex = null; + for (Index i : indices) { + int indexNameCounter = parseIndexNameCounter(i.getName()); if (maxIndexCounter < indexNameCounter) { maxIndexCounter = indexNameCounter; - rolledIndexMeta = indexMetadata; + tmpRolledIndex = i; } } - if (rolledIndexMeta == null) { + if (tmpRolledIndex == null) { return getErrorResultOnNullMetadata(getKey(), index); } - rolledIndexName = rolledIndexMeta.getIndex().getName(); - waitForActiveShardsSettingValue = rolledIndexMeta.getSettings().get("index.write.wait_for_active_shards"); + rolledIndexName = tmpRolledIndex.getName(); + waitForActiveShardsSettingValue = metadata.index(rolledIndexName).getSettings().get("index.write.wait_for_active_shards"); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java index 3bc33a82f8882..564a14150a0c7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java @@ -61,7 +61,7 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener, IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream(); if (dataStream != null) { assert dataStream.getWriteIndex() != null : "datastream " + dataStream.getName() + " has no write index"; - if (dataStream.getWriteIndex().getIndex().equals(index) == false) { + if (dataStream.getWriteIndex().equals(index) == false) { logger.warn("index [{}] is not the write index for data stream [{}]. skipping rollover for policy [{}]", index.getName(), dataStream.getName(), LifecycleSettings.LIFECYCLE_NAME_SETTING.get(metadata.index(index).getSettings())); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 2f92caa6b4d1f..793defdaf4c38 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -17,7 +17,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.cluster.metadata.Metadata; @@ -160,8 +159,7 @@ public static void addDocMappingIfMissing(String alias, listener.onResponse(true); return; } - String[] concreteIndices = indexAbstraction.getIndices().stream().map(IndexMetadata::getIndex).map(Index::getName) - .toArray(String[]::new); + String[] concreteIndices = indexAbstraction.getIndices().stream().map(Index::getName).toArray(String[]::new); String[] indicesThatRequireAnUpdate; try { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index e3ccf45fb3c93..afb2313024acb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -29,10 +29,10 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentType; @@ -132,8 +132,8 @@ public static void createIndexAndAliasIfNecessary(Client client, String firstConcreteIndex = indexPatternPrefix + "-000001"; String[] concreteIndexNames = resolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandHidden(), indexPattern); - Optional indexPointedByCurrentWriteAlias = clusterState.getMetadata().hasAlias(alias) - ? clusterState.getMetadata().getIndicesLookup().get(alias).getIndices().stream().findFirst() + Optional indexPointedByCurrentWriteAlias = clusterState.getMetadata().hasAlias(alias) + ? clusterState.getMetadata().getIndicesLookup().get(alias).getIndices().stream().map(Index::getName).findFirst() : Optional.empty(); if (concreteIndexNames.length == 0) { @@ -149,7 +149,7 @@ public static void createIndexAndAliasIfNecessary(Client client, createFirstConcreteIndex(client, firstConcreteIndex, alias, true, isHiddenAttributeAvailable, indexCreatedListener); return; } - if (indexPointedByCurrentWriteAlias.get().getIndex().getName().equals(legacyIndexWithoutSuffix)) { + if (indexPointedByCurrentWriteAlias.get().equals(legacyIndexWithoutSuffix)) { createFirstConcreteIndex( client, firstConcreteIndex, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java index ec99c2a3eb587..0137e518a2d21 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java @@ -17,6 +17,8 @@ import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.Index; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; @@ -257,10 +259,10 @@ public Collection resolveConcreteIndices() { } else if (indexAbstraction.getType() == IndexAbstraction.Type.CONCRETE_INDEX) { return org.elasticsearch.core.List.of(indexAbstraction.getName()); } else { - final List indices = indexAbstraction.getIndices(); + final List indices = indexAbstraction.getIndices(); final List concreteIndices = new ArrayList<>(indices.size()); for (final IndexMetadata idx : indices) { - concreteIndices.add(idx.getIndex().getName()); + concreteIndices.add(idx.getName()); } return concreteIndices; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java index e846e8e091d73..378f63dac5b8b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java @@ -114,18 +114,34 @@ public void testExceptionThrown() { } public void testPerformActionThrowsExceptionIfIndexIsTheDataStreamWriteIndex() { - String dataStreamName = randomAlphaOfLength(10); - String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); String policyName = "test-ilm-policy"; - IndexMetadata sourceIndexMetadata = - IndexMetadata.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) - .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + String dataStreamName = randomAlphaOfLength(10); + + IndexMetadata index1; + { + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); + index1 = IndexMetadata.builder(indexName) + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)) + .build(); + } + IndexMetadata sourceIndexMetadata; + { + + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2); + sourceIndexMetadata = IndexMetadata.builder(indexName) + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)) + .build(); + } DataStream dataStream = new DataStream(dataStreamName, createTimestampField("@timestamp"), - List.of(sourceIndexMetadata.getIndex())); + List.of(index1.getIndex(), sourceIndexMetadata.getIndex())); ClusterState clusterState = ClusterState.builder(emptyClusterState()).metadata( - Metadata.builder().put(sourceIndexMetadata, true).put(dataStream).build() + Metadata.builder().put(index1, false).put(sourceIndexMetadata, false).put(dataStream).build() ).build(); IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, @@ -144,7 +160,7 @@ public void onFailure(Exception e) { illegalStateException.getMessage(), is( "index [" - + indexName + + sourceIndexMetadata.getIndex().getName() + "] is the write index for data stream [" + dataStreamName + "]. stopping execution of lifecycle [test-ilm-policy] as a data stream's write index cannot be deleted. " diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java index 319799e10650b..716eb6cd171af 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.Index; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.test.ESTestCase; @@ -100,41 +99,44 @@ public void testCalculateUsage() { } { + Metadata.Builder mBuilder = Metadata.builder() + .putCustom(IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata(Collections.singletonMap("mypolicy", + LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), OperationMode.RUNNING)) + .put(IndexMetadata.builder("myindex") + .settings(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy") + .build())) + .put(IndexMetadata.builder("another") + .settings(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy") + .build())) + .put(IndexMetadata.builder("other") + .settings(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, "otherpolicy") + .build())) + + .putCustom(ComposableIndexTemplateMetadata.TYPE, + new ComposableIndexTemplateMetadata(Collections.singletonMap("mytemplate", + new ComposableIndexTemplate(Collections.singletonList("myds"), + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null), + null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false))))); + // Need to get the real Index instance of myindex: + mBuilder.put(new DataStream("myds", new DataStream.TimestampField("@timestamp"), + Collections.singletonList(mBuilder.get("myindex").getIndex()))); + // Test where policy exists and is used by an index, datastream, and template ClusterState state = ClusterState.builder(new ClusterName("mycluster")) - .metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata(Collections.singletonMap("mypolicy", - LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), OperationMode.RUNNING)) - .put(IndexMetadata.builder("myindex") - .settings(Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy") - .build())) - .put(IndexMetadata.builder("another") - .settings(Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy") - .build())) - .put(IndexMetadata.builder("other") - .settings(Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(LifecycleSettings.LIFECYCLE_NAME, "otherpolicy") - .build())) - .put(new DataStream("myds", new DataStream.TimestampField("@timestamp"), - Collections.singletonList(new Index("myindex", "uuid")))) - .putCustom(ComposableIndexTemplateMetadata.TYPE, - new ComposableIndexTemplateMetadata(Collections.singletonMap("mytemplate", - new ComposableIndexTemplate(Collections.singletonList("myds"), - new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null), - null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false))))) - .build()) + .metadata(mBuilder.build()) .build(); assertThat(LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), equalTo(new ItemUsage(Arrays.asList("myindex", "another"), diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DataStreamsStatsTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DataStreamsStatsTransportAction.java index ce5a39c75e2b7..db6bc236a6b4a 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DataStreamsStatsTransportAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DataStreamsStatsTransportAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardsIterator; @@ -118,8 +117,8 @@ protected String[] resolveConcreteIndexNames(ClusterState clusterState, DataStre assert indexAbstraction != null; if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) indexAbstraction; - List indices = dataStream.getIndices(); - return indices.stream().map(idx -> idx.getIndex().getName()); + List indices = dataStream.getIndices(); + return indices.stream().map(Index::getName); } else { return Stream.empty(); } @@ -198,11 +197,7 @@ protected DataStreamsStatsAction.Response newResponse( if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) indexAbstraction; AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats()); - List indices = dataStream.getIndices() - .stream() - .map(IndexMetadata::getIndex) - .map(Index::getName) - .collect(Collectors.toList()); + List indices = dataStream.getIndices().stream().map(Index::getName).collect(Collectors.toList()); stats.backingIndices.addAll(indices); allBackingIndices.addAll(indices); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java index 0b2459834b326..e03ad313f6551 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java @@ -124,7 +124,7 @@ protected CacheKey toKey(SearchRequest searchRequest) { private String getEnrichIndexKey(SearchRequest searchRequest) { String alias = searchRequest.indices()[0]; IndexAbstraction ia = metadata.getIndicesLookup().get(alias); - return ia.getIndices().get(0).getIndex().getName(); + return ia.getIndices().get(0).getName(); } private static class CacheKey { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index 9f0fae534eced..649e3838e6ff5 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -60,7 +60,7 @@ public Processor create(Map processorFactories, Strin } assert indexAbstraction.getType() == IndexAbstraction.Type.ALIAS; assert indexAbstraction.getIndices().size() == 1; - IndexMetadata imd = indexAbstraction.getIndices().get(0); + IndexMetadata imd = metadata.index(indexAbstraction.getIndices().get(0)); Map mappingAsMap = imd.mapping().sourceAsMap(); String policyType = (String) XContentMapValues.extractValue( diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java index cb500c245a8cd..999ef64862cc7 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java @@ -171,9 +171,7 @@ public ClusterState execute(ClusterState currentState) { SortedMap lookup = currentState.metadata().getIndicesLookup(); for (Index index : concreteIndices) { IndexAbstraction ia = lookup.get(index.getName()); - if (ia != null - && ia.getParentDataStream() != null - && ia.getParentDataStream().getWriteIndex().getIndex().equals(index)) { + if (ia != null && ia.getParentDataStream() != null && ia.getParentDataStream().getWriteIndex().equals(index)) { writeIndices.add(index.getName()); } } diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java index e382db4ed29d8..32c4d616057c2 100644 --- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java +++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java @@ -198,7 +198,7 @@ private void logChangedAliasState(IndexAbstraction aliasInfo) { logger.warn("service provider alias [{}] refers to multiple indices [{}] - this is unexpected and is likely to cause problems", ALIAS_NAME, Strings.collectionToCommaDelimitedString(aliasInfo.getIndices())); } else { - logger.info("service provider alias [{}] refers to [{}]", ALIAS_NAME, aliasInfo.getIndices().get(0).getIndex()); + logger.info("service provider alias [{}] refers to [{}]", ALIAS_NAME, aliasInfo.getIndices().get(0)); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index ea1090aec7604..3eb4298e2e381 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexAbstractionResolver; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; @@ -26,6 +25,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.RemoteConnectionStrategy; @@ -327,13 +327,13 @@ static String getPutMappingIndexOrAlias(PutMappingRequest request, Set a .filter(authorizedIndicesList::contains) .filter(aliasName -> { IndexAbstraction alias = metadata.getIndicesLookup().get(aliasName); - List indexMetadata = alias.getIndices(); - if (indexMetadata.size() == 1) { + List indices = alias.getIndices(); + if (indices.size() == 1) { return true; } else { assert alias.getType() == IndexAbstraction.Type.ALIAS; - IndexMetadata idxMeta = alias.getWriteIndex(); - return idxMeta != null && idxMeta.getIndex().getName().equals(concreteIndexName); + Index writeIndex = alias.getWriteIndex(); + return writeIndex != null && writeIndex.getName().equals(concreteIndexName); } }) .findFirst(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index ffbbf7b03fe82..ed0cebe5c4526 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -29,12 +29,12 @@ import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.index.Index; import org.elasticsearch.transport.TransportActionProxy; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; @@ -609,8 +609,8 @@ static Set resolveAuthorizedIndicesFromRole(Role role, RequestInfo reque indicesAndAliases.add(indexAbstraction.getName()); if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { // add data stream and its backing indices for any authorized data streams - for (IndexMetadata indexMetadata : indexAbstraction.getIndices()) { - indicesAndAliases.add(indexMetadata.getIndex().getName()); + for (Index index : indexAbstraction.getIndices()) { + indicesAndAliases.add(index.getName()); } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java index a631d96160811..762548694a901 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java @@ -12,11 +12,11 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.index.Index; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; @@ -188,19 +188,18 @@ private void logDeprecatedPermission(RoleDescriptor roleDescriptor) { final Automaton aliasPrivilegeAutomaton = IndexPrivilege.get(aliasPrivilegeNames).getAutomaton(); final SortedSet inferiorIndexNames = new TreeSet<>(); // check if the alias grants superiors privileges than the indices it points to - for (IndexMetadata indexMetadata : aliasOrIndexMap.get(aliasName).getIndices()) { - final String indexName = indexMetadata.getIndex().getName(); - final Set indexPrivileges = privilegesByIndexMap.get(indexName); + for (Index index : aliasOrIndexMap.get(aliasName).getIndices()) { + final Set indexPrivileges = privilegesByIndexMap.get(index.getName()); // null iff the index does not have *any* privilege if (indexPrivileges != null) { // compute automaton once per index no matter how many times it is pointed to - final Automaton indexPrivilegeAutomaton = indexAutomatonMap.computeIfAbsent(indexName, + final Automaton indexPrivilegeAutomaton = indexAutomatonMap.computeIfAbsent(index.getName(), i -> IndexPrivilege.get(indexPrivileges).getAutomaton()); if (false == Operations.subsetOf(indexPrivilegeAutomaton, aliasPrivilegeAutomaton)) { - inferiorIndexNames.add(indexName); + inferiorIndexNames.add(index.getName()); } } else { - inferiorIndexNames.add(indexName); + inferiorIndexNames.add(index.getName()); } } // log inferior indices for this role, for this alias diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 5ace1f9542959..fba73c5f3fd29 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -54,7 +54,6 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -280,12 +279,11 @@ private static Set loadIndexMappingVersions(String aliasName, ClusterSt private static IndexMetadata resolveConcreteIndex(final String indexOrAliasName, final Metadata metadata) { final IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(indexOrAliasName); if (indexAbstraction != null) { - final List indices = indexAbstraction.getIndices(); + final List indices = indexAbstraction.getIndices(); if (indexAbstraction.getType() != IndexAbstraction.Type.CONCRETE_INDEX && indices.size() > 1) { - throw new IllegalStateException("Alias [" + indexOrAliasName + "] points to more than one index: " + - indices.stream().map(imd -> imd.getIndex().getName()).collect(Collectors.toList())); + throw new IllegalStateException("Alias [" + indexOrAliasName + "] points to more than one index: " + indices); } - return indices.get(0); + return metadata.index(indices.get(0)); } return null; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 1a3a16d2d0505..20e79a85d8225 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -521,7 +521,7 @@ protected void deleteSecurityIndex() { private static Index resolveSecurityIndex(Metadata metadata) { final IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(SECURITY_MAIN_ALIAS); if (indexAbstraction != null) { - return indexAbstraction.getIndices().get(0).getIndex(); + return indexAbstraction.getIndices().get(0); } return null; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java index 917e075161b32..abc164d6c37de 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java @@ -1067,10 +1067,10 @@ public void testBackingIndicesAreIncludedForAuthorizedDataStreams() { } DataStream ds = new DataStream(dataStreamName, null, backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList())); - IndexAbstraction.DataStream iads = new IndexAbstraction.DataStream(ds, backingIndices, emptyList()); + IndexAbstraction.DataStream iads = new IndexAbstraction.DataStream(ds, emptyList()); lookup.put(ds.getName(), iads); for (IndexMetadata im : backingIndices) { - lookup.put(im.getIndex().getName(), new IndexAbstraction.Index(im, iads)); + lookup.put(im.getIndex().getName(), new IndexAbstraction.ConcreteIndex(im, iads)); } SearchRequest request = new SearchRequest("*"); @@ -1100,10 +1100,10 @@ public void testExplicitMappingUpdatesAreNotGrantedWithIngestPrivileges() { } DataStream ds = new DataStream(dataStreamName, null, backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList())); - IndexAbstraction.DataStream iads = new IndexAbstraction.DataStream(ds, backingIndices, emptyList()); + IndexAbstraction.DataStream iads = new IndexAbstraction.DataStream(ds, emptyList()); lookup.put(ds.getName(), iads); for (IndexMetadata im : backingIndices) { - lookup.put(im.getIndex().getName(), new IndexAbstraction.Index(im, iads)); + lookup.put(im.getIndex().getName(), new IndexAbstraction.ConcreteIndex(im, iads)); } PutMappingRequest request = new PutMappingRequest("*"); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java index 0cf973095a356..956b8298794e7 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; @@ -73,8 +74,9 @@ private static void createAuditAliasForDataFrameBWC(ClusterState state, Client c return; } + Metadata metadata = state.metadata(); if (state.getMetadata().getIndicesLookup().get(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED).getIndices().stream() - .anyMatch(metadata -> metadata.getAliases().containsKey(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS))) { + .anyMatch(name -> metadata.index(name).getAliases().containsKey(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS))) { finalListener.onResponse(false); return; } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchStoreUtils.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchStoreUtils.java index 68f32acdad83b..40c4719fae8c1 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchStoreUtils.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchStoreUtils.java @@ -32,7 +32,7 @@ public static IndexMetadata getConcreteIndex(String name, Metadata metadata) { throw new IllegalStateException("Alias [" + name + "] points to more than one index"); } - return indexAbstraction.getIndices().get(0); + return metadata.index(indexAbstraction.getIndices().get(0)); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java index fcf70882e282d..325aeb4d242ed 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java @@ -716,7 +716,7 @@ private ClusterState mockClusterState(String watchIndex) { IndexMetadata indexMetadata = mock(IndexMetadata.class); when(indexMetadata.getIndex()).thenReturn(new Index(watchIndex, randomAlphaOfLength(10))); - indices.put(watchIndex, new IndexAbstraction.Index(indexMetadata)); + indices.put(watchIndex, new IndexAbstraction.ConcreteIndex(indexMetadata)); // now point the alias, if the watch index is not .watches if (watchIndex.equals(Watch.INDEX) == false) { @@ -727,6 +727,7 @@ private ClusterState mockClusterState(String watchIndex) { aliases.put(Watch.INDEX, aliasMetadata); when(indexMetadata.getAliases()).thenReturn(aliases.build()); indices.put(Watch.INDEX, new IndexAbstraction.Alias(aliasMetadata, Collections.singletonList(indexMetadata))); + when(metadata.index(any(Index.class))).thenReturn(indexMetadata); } when(metadata.getIndicesLookup()).thenReturn(indices);