diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index cd199057e1d96..79f458a26411e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -179,6 +179,23 @@ public boolean isSystem() { public List getAliases() { return aliases; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConcreteIndex that = (ConcreteIndex) o; + return isHidden == that.isHidden && + isSystem == that.isSystem && + concreteIndexName.equals(that.concreteIndexName) && + Objects.equals(aliases, that.aliases) && + Objects.equals(dataStream, that.dataStream); + } + + @Override + public int hashCode() { + return Objects.hash(concreteIndexName, isHidden, isSystem, aliases, dataStream); + } } /** @@ -322,6 +339,24 @@ private void validateAliasProperties(List referenceIndexMetadatas private boolean isNonEmpty(List idxMetas) { return (Objects.isNull(idxMetas) || idxMetas.isEmpty()) == false; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Alias alias = (Alias) o; + return isHidden == alias.isHidden && + isSystem == alias.isSystem && + dataStreamAlias == alias.dataStreamAlias && + aliasName.equals(alias.aliasName) && + referenceIndexMetadatas.equals(alias.referenceIndexMetadatas) && + Objects.equals(writeIndex, alias.writeIndex); + } + + @Override + public int hashCode() { + return Objects.hash(aliasName, referenceIndexMetadatas, writeIndex, isHidden, isSystem, dataStreamAlias); + } } class DataStream implements IndexAbstraction { @@ -383,6 +418,20 @@ public List getAliases() { public org.elasticsearch.cluster.metadata.DataStream getDataStream() { return dataStream; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DataStream that = (DataStream) o; + return dataStream.equals(that.dataStream) && + Objects.equals(referencedByDataStreamAliases, that.referencedByDataStreamAliases); + } + + @Override + public int hashCode() { + return Objects.hash(dataStream, referencedByDataStreamAliases); + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index b5d7fcddd91a3..589739d80dafd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1038,15 +1038,18 @@ public static class Builder { private final ImmutableOpenMap.Builder templates; private final ImmutableOpenMap.Builder customs; + private SortedMap previousIndicesLookup; + public Builder() { clusterUUID = UNKNOWN_CLUSTER_UUID; indices = ImmutableOpenMap.builder(); templates = ImmutableOpenMap.builder(); customs = ImmutableOpenMap.builder(); indexGraveyard(IndexGraveyard.builder().build()); // create new empty index graveyard to initialize + previousIndicesLookup = null; } - public Builder(Metadata metadata) { + Builder(Metadata metadata) { this.clusterUUID = metadata.clusterUUID; this.clusterUUIDCommitted = metadata.clusterUUIDCommitted; this.coordinationMetadata = metadata.coordinationMetadata; @@ -1057,13 +1060,17 @@ public Builder(Metadata metadata) { this.indices = ImmutableOpenMap.builder(metadata.indices); this.templates = ImmutableOpenMap.builder(metadata.templates); this.customs = ImmutableOpenMap.builder(metadata.customs); + previousIndicesLookup = metadata.getIndicesLookup(); } public Builder put(IndexMetadata.Builder indexMetadataBuilder) { // we know its a new one, increment the version and store indexMetadataBuilder.version(indexMetadataBuilder.version() + 1); IndexMetadata indexMetadata = indexMetadataBuilder.build(); - indices.put(indexMetadata.getIndex().getName(), indexMetadata); + IndexMetadata previous = indices.put(indexMetadata.getIndex().getName(), indexMetadata); + if (unsetPreviousIndicesLookup(previous, indexMetadata)) { + previousIndicesLookup = null; + } return this; } @@ -1075,10 +1082,37 @@ public Builder put(IndexMetadata indexMetadata, boolean incrementVersion) { if (incrementVersion) { indexMetadata = IndexMetadata.builder(indexMetadata).version(indexMetadata.getVersion() + 1).build(); } - indices.put(indexMetadata.getIndex().getName(), indexMetadata); + IndexMetadata previous = indices.put(indexMetadata.getIndex().getName(), indexMetadata); + if (unsetPreviousIndicesLookup(previous, indexMetadata)) { + previousIndicesLookup = null; + } return this; } + boolean unsetPreviousIndicesLookup(IndexMetadata previous, IndexMetadata current) { + if (previous == null) { + return true; + } + + if (previous.getAliases().equals(current.getAliases()) == false) { + return true; + } + + if (previous.isHidden() != current.isHidden()) { + return true; + } + + if (previous.isSystem() != current.isSystem()) { + return true; + } + + if (previous.getState() != current.getState()) { + return true; + } + + return false; + } + public IndexMetadata get(String index) { return indices.get(index); } @@ -1097,16 +1131,22 @@ public IndexMetadata getSafe(Index index) { } public Builder remove(String index) { + previousIndicesLookup = null; + indices.remove(index); return this; } public Builder removeAllIndices() { + previousIndicesLookup = null; + indices.clear(); return this; } public Builder indices(ImmutableOpenMap indices) { + previousIndicesLookup = null; + this.indices.putAll(indices); return this; } @@ -1187,6 +1227,8 @@ public Builder removeIndexTemplate(String name) { } public DataStream dataStream(String dataStreamName) { + previousIndicesLookup = null; + DataStreamMetadata dataStreamMetadata = (DataStreamMetadata) customs.get(DataStreamMetadata.TYPE); if (dataStreamMetadata != null) { return dataStreamMetadata.dataStreams().get(dataStreamName); @@ -1196,11 +1238,15 @@ public DataStream dataStream(String dataStreamName) { } public Builder dataStreams(Map dataStreams, Map dataStreamAliases) { + previousIndicesLookup = null; + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(dataStreams, dataStreamAliases)); return this; } public Builder put(DataStream dataStream) { + previousIndicesLookup = null; + Objects.requireNonNull(dataStream, "it is invalid to add a null data stream"); Map existingDataStreams = Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) @@ -1217,6 +1263,8 @@ public Builder put(DataStream dataStream) { } public boolean put(String aliasName, String dataStream, Boolean isWriteDataStream, String filter) { + previousIndicesLookup = null; + Map existingDataStream = Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) .map(dsmd -> new HashMap<>(dsmd.dataStreams())) @@ -1255,6 +1303,8 @@ public boolean put(String aliasName, String dataStream, Boolean isWriteDataStrea } public Builder removeDataStream(String name) { + previousIndicesLookup = null; + Map existingDataStreams = Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) .map(dsmd -> new HashMap<>(dsmd.dataStreams())) @@ -1291,6 +1341,8 @@ public Builder removeDataStream(String name) { } public boolean removeDataStreamAlias(String aliasName, String dataStreamName, boolean mustExist) { + previousIndicesLookup = null; + Map dataStreamAliases = Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) @@ -1538,10 +1590,15 @@ public Metadata build(boolean builtIndicesLookupEagerly) { ImmutableOpenMap indices = this.indices.build(); SortedMap indicesLookup; - if (builtIndicesLookupEagerly) { - indicesLookup = Collections.unmodifiableSortedMap(buildIndicesLookup(dataStreamMetadata, indices)); + if (previousIndicesLookup != null) { + assert previousIndicesLookup.equals(buildIndicesLookup(dataStreamMetadata, indices)); + indicesLookup = previousIndicesLookup; } else { - indicesLookup = null; + if (builtIndicesLookupEagerly) { + indicesLookup = Collections.unmodifiableSortedMap(buildIndicesLookup(dataStreamMetadata, indices)); + } else { + indicesLookup = null; + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java index 48fc1ffbe6d3e..8dc6ba0e47031 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java @@ -121,7 +121,9 @@ public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable) } if (metadataBuilder != null) { - return metadataBuilder.build(); + Metadata newMetadata = metadataBuilder.build(); + assert oldMetadata.getIndicesLookup() == newMetadata.getIndicesLookup(); + return newMetadata; } else { return oldMetadata; } diff --git a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java index c1bf8f27ff874..19ef3b0f44d64 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java @@ -368,6 +368,7 @@ private ClusterState patchVersions(ClusterState previousClusterState, ClusterTas if (previousClusterState != newClusterState) { // only the master controls the version numbers + final var previousIndicesLookup = newClusterState.metadata().getIndicesLookup(); Builder builder = incrementVersion(newClusterState); if (previousClusterState.routingTable() != newClusterState.routingTable()) { builder.routingTable(RoutingTable.builder(newClusterState.routingTable()) @@ -378,6 +379,7 @@ private ClusterState patchVersions(ClusterState previousClusterState, ClusterTas } newClusterState = builder.build(); + assert previousIndicesLookup == newClusterState.metadata().getIndicesLookup(); } return newClusterState; diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java index 234a3f8f96e07..f8505cfd26d4c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java @@ -354,7 +354,7 @@ private static ClusterState createNonInitializedState(final int numNodes, final private static ClusterState nextState(final ClusterState previousState, List customMetadataList) { final ClusterState.Builder builder = ClusterState.builder(previousState); builder.stateUUID(UUIDs.randomBase64UUID()); - Metadata.Builder metadataBuilder = new Metadata.Builder(previousState.metadata()); + Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); for (ObjectObjectCursor customMetadata : previousState.metadata().customs()) { if (customMetadata.value instanceof TestCustomMetadata) { metadataBuilder.removeCustom(customMetadata.key); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index fe56fe4bc5acc..f6a676bedd857 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.XContentParser; @@ -57,8 +58,10 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.Matchers.startsWith; public class MetadataTests extends ESTestCase { @@ -1520,6 +1523,102 @@ public void testDataStreamWriteRemoveDataStream() { assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-replicated")); } + public void testReuseIndicesLookup() { + String indexName = "my-index"; + String aliasName = "my-alias"; + String dataStreamName = "logs-mysql-prod"; + String dataStreamAliasName = "logs-mysql"; + Metadata previous = Metadata.builder().build(); + + // Things that should change indices lookup + { + Metadata.Builder builder = Metadata.builder(previous); + IndexMetadata idx = DataStreamTestHelper.createFirstBackingIndex(dataStreamName).build(); + builder.put(idx, true); + DataStream dataStream = new DataStream(dataStreamName, new DataStream.TimestampField("@timestamp"), List.of(idx.getIndex())); + builder.put(dataStream); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(metadata.getIndicesLookup()))); + previous = metadata; + } + { + Metadata.Builder builder = Metadata.builder(previous); + builder.put(dataStreamAliasName, dataStreamName, false, null); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(metadata.getIndicesLookup()))); + previous = metadata; + } + { + Metadata.Builder builder = Metadata.builder(previous); + builder.put(dataStreamAliasName, dataStreamName, true, null); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(metadata.getIndicesLookup()))); + previous = metadata; + } + { + Metadata.Builder builder = Metadata.builder(previous); + builder.put(IndexMetadata.builder(indexName) + .settings(settings(Version.CURRENT)).creationDate(randomNonNegativeLong()) + .numberOfShards(1).numberOfReplicas(0)); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(metadata.getIndicesLookup()))); + previous = metadata; + } + { + Metadata.Builder builder = Metadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + imBuilder.putAlias(AliasMetadata.builder(aliasName).build()); + builder.put(imBuilder); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(metadata.getIndicesLookup()))); + previous = metadata; + } + { + Metadata.Builder builder = Metadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + imBuilder.putAlias(AliasMetadata.builder(aliasName).writeIndex(true).build()); + builder.put(imBuilder); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(metadata.getIndicesLookup()))); + previous = metadata; + } + { + Metadata.Builder builder = Metadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + Settings.Builder sBuilder = Settings.builder() + .put(builder.get(indexName).getSettings()) + .put(IndexMetadata.INDEX_HIDDEN_SETTING.getKey(), true); + imBuilder.settings(sBuilder.build()); + builder.put(imBuilder); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(metadata.getIndicesLookup()))); + previous = metadata; + } + + // Things that shouldn't change indices lookup + { + Metadata.Builder builder = Metadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + imBuilder.numberOfReplicas(2); + builder.put(imBuilder); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), sameInstance(metadata.getIndicesLookup())); + previous = metadata; + } + { + Metadata.Builder builder = Metadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + Settings.Builder sBuilder = Settings.builder() + .put(builder.get(indexName).getSettings()) + .put(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), "val"); + imBuilder.settings(sBuilder.build()); + builder.put(imBuilder); + Metadata metadata = builder.build(); + assertThat(previous.getIndicesLookup(), sameInstance(metadata.getIndicesLookup())); + previous = metadata; + } + } + public static Metadata randomMetadata() { return randomMetadata(1); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java index 81e387473f694..c60d00558cf17 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java @@ -334,7 +334,7 @@ private ClusterState createRecoveryStateAndInitializeAllocations( final TestSnapshotsInfoService snapshotsInfoService ) { DiscoveryNode node1 = newNode("node1"); - Metadata.Builder metadataBuilder = new Metadata.Builder(metadata); + Metadata.Builder metadataBuilder = Metadata.builder(metadata); RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", "randomId")); Set snapshotIndices = new HashSet<>(); diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java index 43086c3579601..5ec3c7d84cb6b 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java @@ -239,7 +239,7 @@ public static DeprecationInfoAction.Response from(ClusterState state, */ private static ClusterState removeSkippedSettings(ClusterState state, String[] indexNames, List skipTheseDeprecatedSettings) { ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(state); - Metadata.Builder metadataBuilder = new Metadata.Builder(state.metadata()); + Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()); metadataBuilder.transientSettings( metadataBuilder.transientSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false)); metadataBuilder.persistentSettings( diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java index 998be8f6181e9..a986cc7ab9ea3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java @@ -179,7 +179,7 @@ public void testIsAbleToRun() { final ClusterState clusterState = csBuilder.build(); assertThat(updater.isAbleToRun(clusterState), is(true)); - metadata = new Metadata.Builder(clusterState.metadata()); + metadata = Metadata.builder(clusterState.metadata()); routingTable = new RoutingTable.Builder(clusterState.routingTable()); if (randomBoolean()) { routingTable.remove(MlConfigIndex.indexName()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java index dfecf00676514..0030c1ddf41b1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java @@ -55,7 +55,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { MlMetaIndex.indexName(), MlConfigIndex.indexName()).size()); - metadata = new Metadata.Builder(cs.metadata()); + metadata = Metadata.builder(cs.metadata()); routingTable = new RoutingTable.Builder(cs.routingTable()); String indexToRemove = randomFrom(resolver.concreteIndexNames(cs, IndicesOptions.lenientExpandOpen(), ".ml-anomalies-shared", diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java index b45d05def523a..9717c93b5ddbe 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java @@ -83,7 +83,7 @@ public static RoutingTable buildIndexRoutingTable(Index index) { */ public static Metadata addAliasToMetadata(Metadata metadata, String indexName) { AliasMetadata aliasMetadata = AliasMetadata.newAliasMetadataBuilder(SECURITY_MAIN_ALIAS).build(); - Metadata.Builder metadataBuilder = new Metadata.Builder(metadata); + Metadata.Builder metadataBuilder = Metadata.builder(metadata); IndexMetadata indexMetadata = metadata.index(indexName); metadataBuilder.put(IndexMetadata.builder(indexMetadata).putAlias(aliasMetadata)); return metadataBuilder.build(); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java index 64c9845c8f848..1cd2b91ed9368 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java @@ -194,7 +194,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { TransformPersistentTasksExecutor.verifyIndicesPrimaryShardsAreActive(cs, TestIndexNameExpressionResolver.newInstance()).size() ); - metadata = new Metadata.Builder(cs.metadata()); + metadata = Metadata.builder(cs.metadata()); routingTable = new RoutingTable.Builder(cs.routingTable()); String indexToRemove = TransformInternalIndexConstants.LATEST_INDEX_NAME; if (randomBoolean()) {