diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index d64edeed9996f..817adad281ed9 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -511,3 +511,101 @@ POST /_aliases } -------------------------------------------------- // TEST[s/^/PUT test\nPUT test2\n/] + +[[aliases-data-streams]] +===== Data stream aliases + +An alias can also point to one or more data streams. To add a data stream to an +alias, specify the stream's name in the `index` parameter. Wildcards (`*`) are +supported. If a wildcard pattern matches both data streams and indices, the +action only uses the matching data streams. + +You can only specify data streams in the `add` and `remove` actions. Aliases +that point to data streams do not support the following parameters: + +* `filter` +* `index_routing` +* `is_write_index` +* `routing` +* `search_routing` + +For example, the following request adds two data streams to the `logs` alias. + +//// +[source,console] +---- +PUT _data_stream/logs-my_app-default + +PUT _data_stream/logs-nginx.access-prod +---- +//// + +[source,console] +---- +POST _aliases +{ + "actions": [ + { "add": { "index": "logs-my_app-default", "alias": "logs" }}, + { "add": { "index": "logs-nginx.access-prod", "alias": "logs" }} + ] +} +---- +// TEST[continued] + +To verify the alias points to both data streams, use the +<>. + +[source,console] +---- +GET logs-*/_alias +---- +// TEST[continued] + +The API returns: + +[source,console-result] +---- +{ + "logs-my_app-default": { + "aliases": { + "logs": {} + } + }, + "logs-nginx.access-prod": { + "aliases": { + "logs": {} + } + } +} +---- + +Use the `remove` action to remove a data stream from an alias. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { "remove": { "index": "logs-my_app-default", "alias": "logs" }} + ] +} + +GET logs-*/_alias +---- +// TEST[continued] + +The get index alias API returns: + +[source,console-result] +---- +{ + "logs-my_app-default": { + "aliases": {} + }, + "logs-nginx.access-prod": { + "aliases": { + "logs": {} + } + } +} +---- diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index a588dc9ebb638..a673a2587c5cf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -548,7 +548,7 @@ public void testAliasNameExistingIndex() throws Exception { InvalidAliasNameException e = expectThrows(InvalidAliasNameException.class, () -> createIndex("test")); - assertThat(e.getMessage(), equalTo("Invalid alias name [index], an index exists with the same name as the alias")); + assertThat(e.getMessage(), equalTo("Invalid alias name [index]: an index or data stream exists with the same name as the alias")); } public void testAliasEmptyName() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 5f428cf9af1ab..4e63aa4496260 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -88,6 +88,27 @@ protected void masterOperation(final IndicesAliasesRequest request, final Cluste // Resolve all the AliasActions into AliasAction instances and gather all the aliases Set aliases = new HashSet<>(); for (AliasActions action : actions) { + List concreteDataStreams = + indexNameExpressionResolver.dataStreamNames(state, request.indicesOptions(), action.indices()); + if (concreteDataStreams.size() != 0) { + switch (action.actionType()) { + case ADD: + for (String dataStreamName : concreteDataStreams) { + finalActions.add(new AliasAction.AddDataStreamAlias(action.aliases()[0], dataStreamName)); + } + break; + case REMOVE: + for (String dataStreamName : concreteDataStreams) { + finalActions.add( + new AliasAction.RemoveDataStreamAlias(action.aliases()[0], dataStreamName, action.mustExist())); + } + break; + default: + throw new IllegalArgumentException("Unsupported action [" + action.actionType() + "]"); + } + continue; + } + final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request.indicesOptions(), false, action.indices()); for (Index concreteIndex : concreteIndices) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java index 2cfffd2699d6e..009e3a9435649 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java @@ -107,4 +107,9 @@ public IndicesOptions indicesOptions() { public ActionRequestValidationException validate() { return null; } + + @Override + public boolean includeDataStreams() { + return true; + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java index f4ba1580fb8bc..a98a4d21132cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java @@ -10,34 +10,48 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.DataStreamAlias; +import org.elasticsearch.cluster.metadata.DataStreamMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Objects; public class GetAliasesResponse extends ActionResponse { private final ImmutableOpenMap> aliases; + private final Map> dataStreamAliases; - public GetAliasesResponse(ImmutableOpenMap> aliases) { + public GetAliasesResponse(ImmutableOpenMap> aliases, Map> dataStreamAliases) { this.aliases = aliases; + this.dataStreamAliases = dataStreamAliases; } public GetAliasesResponse(StreamInput in) throws IOException { super(in); aliases = in.readImmutableMap(StreamInput::readString, i -> i.readList(AliasMetadata::new)); + dataStreamAliases = in.getVersion().onOrAfter(DataStreamMetadata.DATA_STREAM_ALIAS_VERSION) ? + in.readMap(StreamInput::readString, in1 -> in1.readList(DataStreamAlias::new)) : Map.of(); } public ImmutableOpenMap> getAliases() { return aliases; } + public Map> getDataStreamAliases() { + return dataStreamAliases; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(aliases, StreamOutput::writeString, StreamOutput::writeList); + if (out.getVersion().onOrAfter(DataStreamMetadata.DATA_STREAM_ALIAS_VERSION)) { + out.writeMap(dataStreamAliases, StreamOutput::writeString, StreamOutput::writeList); + } } @Override @@ -49,11 +63,12 @@ public boolean equals(Object o) { return false; } GetAliasesResponse that = (GetAliasesResponse) o; - return Objects.equals(aliases, that.aliases); + return Objects.equals(aliases, that.aliases) && + Objects.equals(dataStreamAliases, that.dataStreamAliases); } @Override public int hashCode() { - return Objects.hash(aliases); + return Objects.hash(aliases, dataStreamAliases); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 216241fbde980..e4273fa032012 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -14,6 +14,8 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.DataStreamAlias; +import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -21,6 +23,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; @@ -30,8 +33,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -67,7 +72,7 @@ protected void masterOperation(GetAliasesRequest request, ClusterState state, Ac final SystemIndexAccessLevel systemIndexAccessLevel = indexNameExpressionResolver.getSystemIndexAccessLevel(); ImmutableOpenMap> aliases = state.metadata().findAliases(request, concreteIndices); listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases, state, - systemIndexAccessLevel, threadPool.getThreadContext(), systemIndices))); + systemIndexAccessLevel, threadPool.getThreadContext(), systemIndices), postProcess(request, state))); } /** @@ -80,6 +85,15 @@ static ImmutableOpenMap> postProcess(GetAliasesReque boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0; ImmutableOpenMap.Builder> mapBuilder = ImmutableOpenMap.builder(aliases); for (String index : concreteIndices) { + IndexAbstraction ia = state.metadata().getIndicesLookup().get(index); + assert ia.getType() == IndexAbstraction.Type.CONCRETE_INDEX; + if (ia.getParentDataStream() != null) { + // Don't include backing indices of data streams, + // because it is just noise. Aliases can't refer + // to backing indices directly. + continue; + } + if (aliases.get(index) == null && noAliasesSpecified) { List previous = mapBuilder.put(index, Collections.emptyList()); assert previous == null; @@ -92,6 +106,21 @@ static ImmutableOpenMap> postProcess(GetAliasesReque return finalResponse; } + Map> postProcess(GetAliasesRequest request, ClusterState state) { + Map> result = new HashMap<>(); + boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0; + List requestedDataStreams = + indexNameExpressionResolver.dataStreamNames(state, request.indicesOptions(), request.indices()); + for (String requestedDataStream : requestedDataStreams) { + List aliases = state.metadata().dataStreamAliases().values().stream() + .filter(alias -> alias.getDataStreams().contains(requestedDataStream)) + .filter(alias -> noAliasesSpecified || Regex.simpleMatch(request.aliases(), alias.getName())) + .collect(Collectors.toList()); + result.put(requestedDataStream, aliases); + } + return result; + } + private static void checkSystemIndexAccess(GetAliasesRequest request, SystemIndices systemIndices, ClusterState state, ImmutableOpenMap> aliasesMap, SystemIndexAccessLevel systemIndexAccessLevel, ThreadContext threadContext) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 46bd1822587e1..3b70fd628ac26 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -560,10 +560,9 @@ private static void enrichIndexAbstraction(String indexAbstraction, SortedMap i.getIndex().getName()).toArray(String[]::new); + String[] indexNames = ia.getIndices().stream().map(i -> i.getIndex().getName()).toArray(String[]::new); Arrays.sort(indexNames); - aliases.add(new ResolvedAlias(alias.getName(), indexNames)); + aliases.add(new ResolvedAlias(ia.getName(), indexNames)); break; case DATA_STREAM: IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) ia; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java index a837a7672b3b6..426e6bd393b71 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java @@ -198,4 +198,60 @@ boolean apply(NewAliasValidator aliasValidator, Metadata.Builder metadata, Index throw new UnsupportedOperationException(); } } + + public static class AddDataStreamAlias extends AliasAction { + + private final String aliasName; + private final String dataStreamName; + + public AddDataStreamAlias(String aliasName, String dataStreamName) { + super(dataStreamName); + this.aliasName = aliasName; + this.dataStreamName = dataStreamName; + } + + public String getAliasName() { + return aliasName; + } + + public String getDataStreamName() { + return dataStreamName; + } + + @Override + boolean removeIndex() { + return false; + } + + @Override + boolean apply(NewAliasValidator aliasValidator, Metadata.Builder metadata, IndexMetadata index) { + aliasValidator.validate(aliasName, null, null, null); + return metadata.put(aliasName, dataStreamName); + } + } + + public static class RemoveDataStreamAlias extends AliasAction { + + private final String aliasName; + private final Boolean mustExist; + private final String dataStreamName; + + public RemoveDataStreamAlias(String aliasName, String dataStreamName, Boolean mustExist) { + super(dataStreamName); + this.aliasName = aliasName; + this.mustExist = mustExist; + this.dataStreamName = dataStreamName; + } + + @Override + boolean removeIndex() { + return false; + } + + @Override + boolean apply(NewAliasValidator aliasValidator, Metadata.Builder metadata, IndexMetadata index) { + boolean mustExist = this.mustExist != null ? this.mustExist : false; + return metadata.removeDataStreamAlias(aliasName, dataStreamName, mustExist); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java index cb8e3aaa7eac1..cf7a091a8b751 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Optional; import java.util.function.Function; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -39,7 +40,7 @@ public class AliasValidator { * @throws IllegalArgumentException if the alias is not valid */ public void validateAlias(Alias alias, String index, Metadata metadata) { - validateAlias(alias.name(), index, alias.indexRouting(), metadata::index); + validateAlias(alias.name(), index, alias.indexRouting(), lookup(metadata)); } /** @@ -48,7 +49,7 @@ public void validateAlias(Alias alias, String index, Metadata metadata) { * @throws IllegalArgumentException if the alias is not valid */ public void validateAliasMetadata(AliasMetadata aliasMetadata, String index, Metadata metadata) { - validateAlias(aliasMetadata.alias(), index, aliasMetadata.indexRouting(), metadata::index); + validateAlias(aliasMetadata.alias(), index, aliasMetadata.indexRouting(), lookup(metadata)); } /** @@ -72,16 +73,16 @@ public void validateAliasStandalone(Alias alias) { /** * Validate a proposed alias. */ - public void validateAlias(String alias, String index, @Nullable String indexRouting, Function indexLookup) { + public void validateAlias(String alias, String index, @Nullable String indexRouting, Function lookup) { validateAliasStandalone(alias, indexRouting); if (Strings.hasText(index) == false) { throw new IllegalArgumentException("index name is required"); } - IndexMetadata indexNamedSameAsAlias = indexLookup.apply(alias); - if (indexNamedSameAsAlias != null) { - throw new InvalidAliasNameException(indexNamedSameAsAlias.getIndex(), alias, "an index exists with the same name as the alias"); + String sameNameAsAlias = lookup.apply(alias); + if (sameNameAsAlias != null) { + throw new InvalidAliasNameException(alias, "an index or data stream exists with the same name as the alias"); } } @@ -134,4 +135,10 @@ private static void validateAliasFilter(XContentParser parser, SearchExecutionCo QueryBuilder queryBuilder = Rewriteable.rewrite(parseInnerQueryBuilder, searchExecutionContext, true); queryBuilder.toQuery(searchExecutionContext); } + + private static Function lookup(Metadata metadata) { + return name -> Optional.ofNullable(metadata.getIndicesLookup().get(name)) + .filter(indexAbstraction -> indexAbstraction.getType() != IndexAbstraction.Type.ALIAS) + .map(IndexAbstraction::getName).orElse(null); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java new file mode 100644 index 0000000000000..9bf8367724ea5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class DataStreamAlias extends AbstractDiffable implements ToXContentObject { + + public static final ParseField DATA_STREAMS_FIELD = new ParseField("data_streams"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "data_stream_alias", + false, + (args, name) -> new DataStreamAlias(name, (List) args[0]) + ); + + static { + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), DATA_STREAMS_FIELD); + } + + private final String name; + private final List dataStreams; + + public DataStreamAlias(String name, List dataStreams) { + this.name = Objects.requireNonNull(name); + this.dataStreams = List.copyOf(dataStreams); + } + + public DataStreamAlias(StreamInput in) throws IOException { + this.name = in.readString(); + this.dataStreams = in.readStringList(); + } + + public String getName() { + return name; + } + + public List getDataStreams() { + return dataStreams; + } + + public static Diff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(DataStreamAlias::new, in); + } + + public static DataStreamAlias fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + if (token != XContentParser.Token.FIELD_NAME) { + throw new ParsingException(parser.getTokenLocation(), "unexpected token"); + } + String name = parser.currentName(); + DataStreamAlias alias = PARSER.parse(parser, name); + return alias; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name); + builder.field(DATA_STREAMS_FIELD.getPreferredName(), dataStreams); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeStringCollection(dataStreams); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DataStreamAlias that = (DataStreamAlias) o; + return Objects.equals(name, that.name) && + Objects.equals(dataStreams, that.dataStreams); + } + + @Override + public int hashCode() { + return Objects.hash(name, dataStreams); + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java index f3164a6bc5219..278d3d46eb3be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java @@ -34,9 +34,17 @@ public class DataStreamMetadata implements Metadata.Custom { public static final String TYPE = "data_stream"; private static final ParseField DATA_STREAM = new ParseField("data_stream"); + private static final ParseField DATA_STREAM_ALIASES = new ParseField("data_stream_aliases"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(TYPE, false, - a -> new DataStreamMetadata((Map) a[0])); + args -> { + Map dataStreams = (Map) args[0]; + Map dataStreamAliases = (Map) args[1]; + if (dataStreamAliases == null) { + dataStreamAliases = Map.of(); + } + return new DataStreamMetadata(dataStreams, dataStreamAliases); + }); static { PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { @@ -47,22 +55,40 @@ public class DataStreamMetadata implements Metadata.Custom { } return dataStreams; }, DATA_STREAM); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + Map dataStreams = new HashMap<>(); + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + DataStreamAlias alias = DataStreamAlias.fromXContent(p); + dataStreams.put(alias.getName(), alias); + } + return dataStreams; + }, DATA_STREAM_ALIASES); } + public static final Version DATA_STREAM_ALIAS_VERSION = Version.V_8_0_0; + private final Map dataStreams; + private final Map dataStreamAliases; - public DataStreamMetadata(Map dataStreams) { + public DataStreamMetadata(Map dataStreams, + Map dataStreamAliases) { this.dataStreams = Collections.unmodifiableMap(new HashMap<>(dataStreams)); + this.dataStreamAliases = Map.copyOf(dataStreamAliases); } public DataStreamMetadata(StreamInput in) throws IOException { - this(in.readMap(StreamInput::readString, DataStream::new)); + this(in.readMap(StreamInput::readString, DataStream::new), in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION) ? + in.readMap(StreamInput::readString, DataStreamAlias::new) : Map.of()); } public Map dataStreams() { return this.dataStreams; } + public Map getDataStreamAliases() { + return dataStreamAliases; + } + @Override public Diff diff(Metadata.Custom before) { return new DataStreamMetadata.DataStreamMetadataDiff((DataStreamMetadata) before, this); @@ -90,6 +116,9 @@ public Version getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(this.dataStreams, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + out.writeMap(this.dataStreamAliases, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + } } public static DataStreamMetadata fromXContent(XContentParser parser) throws IOException { @@ -103,12 +132,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(dataStream.getKey(), dataStream.getValue()); } builder.endObject(); + builder.startObject(DATA_STREAM_ALIASES.getPreferredName()); + for (Map.Entry dataStream : dataStreamAliases.entrySet()) { + dataStream.getValue().toXContent(builder, params); + } + builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(this.dataStreams); + return Objects.hash(this.dataStreams, dataStreamAliases); } @Override @@ -120,7 +154,8 @@ public boolean equals(Object obj) { return false; } DataStreamMetadata other = (DataStreamMetadata) obj; - return Objects.equals(this.dataStreams, other.dataStreams); + return Objects.equals(this.dataStreams, other.dataStreams) && + Objects.equals(this.dataStreamAliases, other.dataStreamAliases); } @Override @@ -131,25 +166,40 @@ public String toString() { static class DataStreamMetadataDiff implements NamedDiff { final Diff> dataStreamDiff; + final Diff> dataStreamAliasDiff; DataStreamMetadataDiff(DataStreamMetadata before, DataStreamMetadata after) { this.dataStreamDiff = DiffableUtils.diff(before.dataStreams, after.dataStreams, DiffableUtils.getStringKeySerializer()); + this.dataStreamAliasDiff = DiffableUtils.diff(before.dataStreamAliases, after.dataStreamAliases, + DiffableUtils.getStringKeySerializer()); } DataStreamMetadataDiff(StreamInput in) throws IOException { this.dataStreamDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DataStream::new, DataStream::readDiffFrom); + if (in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + this.dataStreamAliasDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), + DataStreamAlias::new, DataStreamAlias::readDiffFrom); + } else { + this.dataStreamAliasDiff = null; + } } @Override public Metadata.Custom apply(Metadata.Custom part) { - return new DataStreamMetadata(dataStreamDiff.apply(((DataStreamMetadata) part).dataStreams)); + return new DataStreamMetadata( + dataStreamDiff.apply(((DataStreamMetadata) part).dataStreams), + dataStreamAliasDiff != null ? dataStreamAliasDiff.apply(((DataStreamMetadata) part).dataStreamAliases) : Map.of() + ); } @Override public void writeTo(StreamOutput out) throws IOException { dataStreamDiff.writeTo(out); + if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + dataStreamAliasDiff.writeTo(out); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index f1f71b62375b2..91c90d6d2b6c8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -303,4 +303,55 @@ public org.elasticsearch.cluster.metadata.DataStream getDataStream() { return dataStream; } } + + class DataStreamAlias implements IndexAbstraction { + + private final org.elasticsearch.cluster.metadata.DataStreamAlias dataStreamAlias; + private final List indicesOfAllDataStreams; + + public DataStreamAlias(org.elasticsearch.cluster.metadata.DataStreamAlias dataStreamAlias, + List indicesOfAllDataStreams) { + this.dataStreamAlias = dataStreamAlias; + this.indicesOfAllDataStreams = indicesOfAllDataStreams; + } + + @Override + public Type getType() { + return Type.ALIAS; + } + + @Override + public String getName() { + return dataStreamAlias.getName(); + } + + @Override + public List getIndices() { + return indicesOfAllDataStreams; + } + + @Override + public IndexMetadata getWriteIndex() { + return null; + } + + @Override + public DataStream getParentDataStream() { + return null; + } + + @Override + public boolean isHidden() { + return false; + } + + @Override + public boolean isSystem() { + return false; + } + + public org.elasticsearch.cluster.metadata.DataStreamAlias getDataStreamAlias() { + return dataStreamAlias; + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 73acca17b1b5c..bebb87698e834 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -141,8 +141,11 @@ public List dataStreamNames(ClusterState state, IndicesOptions options, indexExpressions = new String[]{"*"}; } - List dataStreams = wildcardExpressionResolver.resolve(context, Arrays.asList(indexExpressions)); - return ((dataStreams == null) ? org.elasticsearch.common.collect.List.of() : dataStreams).stream() + List expressions = Arrays.asList(indexExpressions); + for (ExpressionResolver expressionResolver : expressionResolvers) { + expressions = expressionResolver.resolve(context, expressions); + } + return ((expressions == null) ? org.elasticsearch.common.collect.List.of() : expressions).stream() .map(x -> state.metadata().getIndicesLookup().get(x)) .filter(Objects::nonNull) .filter(ia -> ia.getType() == IndexAbstraction.Type.DATA_STREAM) @@ -594,7 +597,7 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab String concreteIndex = index.getIndex().getName(); AliasMetadata aliasMetadata = index.getAliases().get(indexAbstraction.getName()); if (norouting.contains(concreteIndex) == false) { - if (aliasMetadata.searchRoutingValues().isEmpty() == false) { + if (aliasMetadata != null && aliasMetadata.searchRoutingValues().isEmpty() == false) { // Routing alias if (routings == null) { routings = new HashMap<>(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 5b028a8bdf0f0..830260478ebc2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -15,6 +15,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.cluster.ClusterState; @@ -277,6 +278,19 @@ public boolean equalsAliases(Metadata other) { } } + if (other.dataStreamAliases().size() != dataStreamAliases().size()) { + return false; + } + for (DataStreamAlias otherAlias : other.dataStreamAliases().values()) { + DataStreamAlias thisAlias = dataStreamAliases().get(otherAlias.getName()); + if (thisAlias == null) { + return false; + } + if (thisAlias.equals(otherAlias) == false) { + return false; + } + } + return true; } @@ -611,22 +625,12 @@ public String resolveWriteIndexRouting(@Nullable String routing, String aliasOrI if (writeIndex == null) { throw new IllegalArgumentException("alias [" + aliasOrIndex + "] does not have a write index"); } - AliasMetadata aliasMd = writeIndex.getAliases().get(result.getName()); - if (aliasMd.indexRouting() != null) { - if (aliasMd.indexRouting().indexOf(',') != -1) { - throw new IllegalArgumentException("index/alias [" + aliasOrIndex + "] provided with routing value [" - + aliasMd.getIndexRouting() + "] that resolved to several routing values, rejecting operation"); - } - if (routing != null) { - if (routing.equals(aliasMd.indexRouting()) == false) { - throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has index routing associated with it [" - + aliasMd.indexRouting() + "], and was provided with routing value [" + routing + "], rejecting operation"); - } - } - // Alias routing overrides the parent routing (if any). - return aliasMd.indexRouting(); + AliasMetadata writeIndexAliasMetadata = writeIndex.getAliases().get(result.getName()); + if (writeIndexAliasMetadata != null) { + return resolveRouting(routing, aliasOrIndex, writeIndexAliasMetadata); + } else { + return routing; } - return routing; } /** @@ -755,6 +759,12 @@ public Map dataStreams() { .orElse(Collections.emptyMap()); } + public Map dataStreamAliases() { + return Optional.ofNullable((DataStreamMetadata) this.custom(DataStreamMetadata.TYPE)) + .map(DataStreamMetadata::getDataStreamAliases) + .orElse(Collections.emptyMap()); + } + public ImmutableOpenMap customs() { return this.customs; } @@ -1251,11 +1261,18 @@ public Builder removeIndexTemplate(String name) { } public DataStream dataStream(String dataStreamName) { - return ((DataStreamMetadata) customs.get(DataStreamMetadata.TYPE)).dataStreams().get(dataStreamName); + DataStreamMetadata dataStreamMetadata = (DataStreamMetadata) customs.get(DataStreamMetadata.TYPE); + if (dataStreamMetadata != null) { + return dataStreamMetadata.dataStreams().get(dataStreamName); + } else { + return null; + } } public Builder dataStreams(Map dataStreams) { - this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(dataStreams)); + // TODO: take into account aliases... + // (this is only used from snapshot / restore) + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(dataStreams, new HashMap<>())); return this; } @@ -1266,20 +1283,111 @@ public Builder put(DataStream dataStream) { .map(dsmd -> new HashMap<>(dsmd.dataStreams())) .orElse(new HashMap<>()); existingDataStreams.put(dataStream.getName(), dataStream); - this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams)); + Map existingDataStreamAliases = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) + .orElse(new HashMap<>()); + + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams, existingDataStreamAliases)); return this; } + public boolean put(String name, String dataStream) { + Map existingDataStream = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.dataStreams())) + .orElse(new HashMap<>()); + Map dataStreamAliases = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) + .orElse(new HashMap<>()); + + if (existingDataStream.containsKey(dataStream) == false) { + throw new IllegalArgumentException("alias [" + name + "] refers to a non existing data stream [" + dataStream + "]"); + } + + DataStreamAlias alias = dataStreamAliases.get(name); + if (alias == null) { + alias = new DataStreamAlias(name, List.of(dataStream)); + } else { + Set dataStreams = new HashSet<>(alias.getDataStreams()); + boolean added = dataStreams.add(dataStream); + if (added == false) { + return false; + } + alias = new DataStreamAlias(name, List.copyOf(dataStreams)); + } + dataStreamAliases.put(name, alias); + + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStream, dataStreamAliases)); + return true; + } + public Builder removeDataStream(String name) { Map existingDataStreams = Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) .map(dsmd -> new HashMap<>(dsmd.dataStreams())) .orElse(new HashMap<>()); existingDataStreams.remove(name); - this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams)); + + Map existingDataStreamAliases = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) + .orElse(new HashMap<>()); + + Set aliasesToDelete = new HashSet<>(); + List aliasesToUpdate = new ArrayList<>(); + for (var alias : existingDataStreamAliases.values()) { + Set dataStreams = new HashSet<>(alias.getDataStreams()); + if (dataStreams.contains(name)) { + dataStreams.remove(name); + if (dataStreams.isEmpty()) { + aliasesToDelete.add(alias.getName()); + } else { + aliasesToUpdate.add(new DataStreamAlias(alias.getName(), List.copyOf(dataStreams))); + } + } + } + for (DataStreamAlias alias : aliasesToUpdate) { + existingDataStreamAliases.put(alias.getName(), alias); + } + for (String aliasToDelete : aliasesToDelete) { + existingDataStreamAliases.remove(aliasToDelete); + } + + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams, existingDataStreamAliases)); return this; } + public boolean removeDataStreamAlias(String aliasName, String dataStreamName, boolean mustExist) { + Map dataStreamAliases = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) + .orElse(new HashMap<>()); + + DataStreamAlias existing = dataStreamAliases.get(aliasName); + if (mustExist && existing == null) { + throw new ResourceNotFoundException("alias [" + aliasName + "] doesn't exist"); + } else if (existing == null) { + return false; + } + Set dataStreams = new HashSet<>(existing.getDataStreams()); + dataStreams.remove(dataStreamName); + if (dataStreams.isEmpty()) { + dataStreamAliases.remove(aliasName); + } else { + dataStreamAliases.put(aliasName, + new DataStreamAlias(existing.getName(), List.copyOf(dataStreams))); + } + + Map existingDataStream = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.dataStreams())) + .orElse(new HashMap<>()); + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStream, dataStreamAliases)); + return true; + } + public Custom getCustom(String type) { return customs.get(type); } @@ -1527,6 +1635,16 @@ private SortedMap buildIndicesLookup() { indexToDataStreamLookup.put(i.getName(), dataStream); } } + for (DataStreamAlias alias : dataStreamMetadata.getDataStreamAliases().values()) { + List allIndicesOfAllDataStreams = alias.getDataStreams().stream() + .map(name -> dataStreamMetadata.dataStreams().get(name)) + .flatMap(ds -> ds.getIndices().stream()) + .map(index -> indices.get(index.getName())) + .collect(Collectors.toList()); + IndexAbstraction existing = indicesLookup.put(alias.getName(), + new IndexAbstraction.DataStreamAlias(alias, allIndicesOfAllDataStreams)); + assert existing == null : "duplicate data stream alias for " + alias.getName(); + } } Map> aliasToIndices = new HashMap<>(); @@ -1575,6 +1693,7 @@ static void validateDataStreams(SortedMap indicesLooku // Sanity check, because elsewhere a more user friendly error should have occurred: List conflictingAliases = indicesLookup.values().stream() .filter(ia -> ia.getType() == IndexAbstraction.Type.ALIAS) + .filter(ia -> ia instanceof IndexAbstraction.Alias) // TODO: a nicer to only select index aliases? .filter(ia -> { for (IndexMetadata index : ia.getIndices()) { if (indicesLookup.get(index.getIndex().getName()).getParentDataStream() != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index d1d7b2049b0b6..8ff7a4a9a7aef 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -110,16 +110,40 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable lookup = name -> { + IndexMetadata imd = metadata.get(name); + if (imd != null) { + return imd.getIndex().getName(); + } + DataStream dataStream = metadata.dataStream(name); + if (dataStream != null) { + return dataStream.getName(); + } + return null; + }; + + // Handle the actions that do data streams aliases separately: + DataStream dataStream = metadata.dataStream(action.getIndex()); + if (dataStream != null) { + NewAliasValidator newAliasValidator = (alias, indexRouting, filter, writeIndex) -> { + aliasValidator.validateAlias(alias, action.getIndex(), indexRouting, lookup); + }; + if (action.apply(newAliasValidator, metadata, null)) { + changed = true; + } + continue; + } + IndexMetadata index = metadata.get(action.getIndex()); if (index == null) { throw new IndexNotFoundException(action.getIndex()); } validateAliasTargetIsNotDSBackingIndex(currentState, action); NewAliasValidator newAliasValidator = (alias, indexRouting, filter, writeIndex) -> { - /* It is important that we look up the index using the metadata builder we are modifying so we can remove an - * index and replace it with an alias. */ - Function indexLookup = name -> metadata.get(name); - aliasValidator.validateAlias(alias, action.getIndex(), indexRouting, indexLookup); + aliasValidator.validateAlias(alias, action.getIndex(), indexRouting, lookup); if (Strings.hasLength(filter)) { IndexService indexService = indices.get(index.getIndex().getName()); if (indexService == null) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java index 09a39dd5b6b2d..3a0edc6199415 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -31,6 +32,7 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -65,7 +67,7 @@ public String getName() { static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String[] requestedAliases, ImmutableOpenMap> responseAliasMap, - XContentBuilder builder) throws Exception { + Map> dataStreamAliases, XContentBuilder builder) throws Exception { final Set indicesToDisplay = new HashSet<>(); final Set returnedAliasNames = new HashSet<>(); for (final ObjectObjectCursor> cursor : responseAliasMap) { @@ -132,7 +134,7 @@ static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String builder.field("status", status.getStatus()); } - for (final ObjectObjectCursor> entry : responseAliasMap) { + for (final var entry : responseAliasMap) { if (aliasesExplicitlyRequested == false || (aliasesExplicitlyRequested && indicesToDisplay.contains(entry.key))) { builder.startObject(entry.key); { @@ -147,6 +149,20 @@ static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String builder.endObject(); } } + for (var entry : dataStreamAliases.entrySet()) { + builder.startObject(entry.getKey()); + { + builder.startObject("aliases"); + { + for (DataStreamAlias alias : entry.getValue()) { + builder.startObject(alias.getName()); + builder.endObject(); + } + } + builder.endObject(); + } + builder.endObject(); + } } builder.endObject(); return new BytesRestResponse(status, builder); @@ -171,7 +187,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(GetAliasesResponse response, XContentBuilder builder) throws Exception { - return buildRestResponse(namesProvided, aliases, response.getAliases(), builder); + return buildRestResponse(namesProvided, aliases, response.getAliases(), response.getDataStreamAliases(), builder); } }); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java index ba2fd776e215b..ce801ffc57f94 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.alias.RandomAliasActionsGenerator; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -26,7 +27,6 @@ import java.util.Objects; import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomAliasAction; -import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomMap; import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomRouting; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.arrayContaining; @@ -111,7 +111,7 @@ public void testMustExistOption() { public void testParseAdd() throws IOException { String[] indices = generateRandomStringArray(10, 5, false, false); String[] aliases = generateRandomStringArray(10, 5, false, false); - Map filter = randomBoolean() ? randomMap(5) : null; + Map filter = randomBoolean() ? RandomAliasActionsGenerator.randomMap(5) : null; Object searchRouting = randomBoolean() ? randomRouting() : null; Object indexRouting = randomBoolean() ? randomBoolean() ? searchRouting : randomRouting() : null; boolean writeIndex = randomBoolean(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java index 8973bf236f705..cd6d7c94427e7 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java @@ -8,9 +8,11 @@ package org.elasticsearch.action.admin.indices.alias.get; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.AliasMetadata.Builder; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -34,7 +36,8 @@ protected Writeable.Reader instanceReader() { @Override protected GetAliasesResponse mutateInstance(GetAliasesResponse response) { - return new GetAliasesResponse(mutateAliases(response.getAliases())); + return new GetAliasesResponse(mutateAliases(response.getAliases()), + randomMap(5, 5, () -> new Tuple<>(randomAlphaOfLength(4), randomList(5, DataStreamTestHelper::randomAliasInstance)))); } private static ImmutableOpenMap> mutateAliases(ImmutableOpenMap> aliases) { @@ -75,7 +78,8 @@ private static ImmutableOpenMap> mutateAliases(Immut } private static GetAliasesResponse createTestItem() { - return new GetAliasesResponse(createIndicesAliasesMap(0, 5).build()); + return new GetAliasesResponse(mutateAliases(createIndicesAliasesMap(0, 5).build() + ), randomMap(5, 5, () -> new Tuple<>(randomAlphaOfLength(4), randomList(5, DataStreamTestHelper::randomAliasInstance)))); } private static ImmutableOpenMap.Builder> createIndicesAliasesMap(int min, int max) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index f8565133bb571..229de26d68713 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -29,12 +29,19 @@ public class TransportGetAliasesActionTests extends ESTestCase { public void testPostProcess() { + Metadata.Builder metadata = Metadata.builder(); + metadata.put(IndexMetadata.builder("a").settings(ESTestCase.settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)); + metadata.put(IndexMetadata.builder("b").settings(ESTestCase.settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)); + metadata.put(IndexMetadata.builder("c").settings(ESTestCase.settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(metadata).build(); + GetAliasesRequest request = new GetAliasesRequest(); ImmutableOpenMap> aliases = ImmutableOpenMap.>builder() .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build())) .build(); ImmutableOpenMap> result = - TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, + TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, clusterState, SystemIndexAccessLevel.NONE, null, EmptySystemIndices.INSTANCE); assertThat(result.size(), equalTo(3)); assertThat(result.get("a").size(), equalTo(0)); @@ -46,7 +53,7 @@ public void testPostProcess() { aliases = ImmutableOpenMap.>builder() .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build())) .build(); - result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, + result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, clusterState, SystemIndexAccessLevel.NONE, null, EmptySystemIndices.INSTANCE); assertThat(result.size(), equalTo(3)); assertThat(result.get("a").size(), equalTo(0)); @@ -57,7 +64,7 @@ public void testPostProcess() { aliases = ImmutableOpenMap.>builder() .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build())) .build(); - result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, + result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, clusterState, SystemIndexAccessLevel.NONE, null, EmptySystemIndices.INSTANCE); assertThat(result.size(), equalTo(1)); assertThat(result.get("b").size(), equalTo(1)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index 6902db27b2152..5a598aee80bbd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -139,7 +139,7 @@ private static ComposableIndexTemplate.DataStreamTemplate randomDataStreamTempla if (randomBoolean()) { return null; } else { - return new ComposableIndexTemplate.DataStreamTemplate(); + return DataStreamTemplateTests.randomInstance(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java index 7e081e2a6f528..7f43a85b18f16 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.test.AbstractNamedWriteableTestCase; import java.io.IOException; @@ -16,18 +17,34 @@ import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + public class DataStreamMetadataTests extends AbstractNamedWriteableTestCase { + public void testFromXContent() throws IOException { + xContentTester(this::createParser, this::createTestInstance, ToXContent.EMPTY_PARAMS, DataStreamMetadata::fromXContent) + .assertEqualsConsumer(this::assertEqualInstances) + .test(); + } + @Override protected DataStreamMetadata createTestInstance() { if (randomBoolean()) { - return new DataStreamMetadata(Collections.emptyMap()); + return new DataStreamMetadata(Map.of(), Map.of()); } Map dataStreams = new HashMap<>(); for (int i = 0; i < randomIntBetween(1, 5); i++) { dataStreams.put(randomAlphaOfLength(5), DataStreamTestHelper.randomInstance()); } - return new DataStreamMetadata(dataStreams); + + Map dataStreamsAliases = new HashMap<>(); + if (randomBoolean()) { + for (int i = 0; i < randomIntBetween(1, 5); i++) { + DataStreamAlias alias = DataStreamTestHelper.randomAliasInstance(); + dataStreamsAliases.put(alias.getName(), alias); + } + } + return new DataStreamMetadata(dataStreams, dataStreamsAliases); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java new file mode 100644 index 0000000000000..65e6efcdca6e4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.cluster.metadata; + +import java.io.IOException; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +public class DataStreamTemplateTests extends AbstractSerializingTestCase { + + @Override + protected DataStreamTemplate doParseInstance(XContentParser parser) throws IOException { + return DataStreamTemplate.PARSER.parse(parser, null); + } + + @Override + protected Writeable.Reader instanceReader() { + return DataStreamTemplate::new; + } + + @Override + protected DataStreamTemplate createTestInstance() { + return randomInstance(); + } + + public static DataStreamTemplate randomInstance() { + return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean()); + } + +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java index 1e966547feaee..2427eff63b63b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.Index; @@ -30,9 +31,12 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anySetOf; @@ -499,6 +503,30 @@ public void testAliasesForDataStreamBackingIndicesNotSupported() { "stream [foo-stream]. Data streams and their backing indices don't support alias operations.")); } + public void testDataStreamAliases() { + ClusterState state = DataStreamTestHelper.getClusterStateWithDataStreams(List.of( + new Tuple<>("logs-foobar", 1), new Tuple<>("metrics-foobar", 1)), List.of()); + + ClusterState result = service.applyAliasActions(state, List.of( + new AliasAction.AddDataStreamAlias("foobar", "logs-foobar"), + new AliasAction.AddDataStreamAlias("foobar", "metrics-foobar") + )); + assertThat(result.metadata().dataStreamAliases().get("foobar"), notNullValue()); + assertThat(result.metadata().dataStreamAliases().get("foobar").getDataStreams(), + containsInAnyOrder("logs-foobar", "metrics-foobar")); + + result = service.applyAliasActions(result, List.of( + new AliasAction.RemoveDataStreamAlias("foobar", "logs-foobar", null) + )); + assertThat(result.metadata().dataStreamAliases().get("foobar"), notNullValue()); + assertThat(result.metadata().dataStreamAliases().get("foobar").getDataStreams(), containsInAnyOrder("metrics-foobar")); + + result = service.applyAliasActions(result, List.of( + new AliasAction.RemoveDataStreamAlias("foobar", "metrics-foobar", null) + )); + assertThat(result.metadata().dataStreamAliases().get("foobar"), nullValue()); + } + private ClusterState applyHiddenAliasMix(ClusterState before, Boolean isHidden1, Boolean isHidden2) { return service.applyAliasActions(before, Arrays.asList( new AliasAction.Add("test", "alias", null, null, null, null, isHidden1), diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 959f907e54e38..2ee90b063bc25 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.cluster.ClusterModule; @@ -50,11 +51,13 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.Metadata.Builder.validateDataStreams; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; public class MetadataTests extends ESTestCase { @@ -1228,7 +1231,7 @@ public void testValidateDataStreamsAllowsPrefixedBackingIndices() { indicesLookup.put(indexMeta.getIndex().getName(), new IndexAbstraction.Index(indexMeta, dataStreamAbstraction)); } } - DataStreamMetadata dataStreamMetadata = new DataStreamMetadata(org.elasticsearch.common.collect.Map.of(dataStreamName, dataStream)); + DataStreamMetadata dataStreamMetadata = new DataStreamMetadata(org.elasticsearch.common.collect.Map.of(dataStreamName, dataStream), Map.of()); // prefixed indices with a lower generation than the data stream's generation are allowed even if the non-prefixed, matching the // data stream backing indices naming pattern, indices are already in the system @@ -1297,6 +1300,118 @@ public void testSnapshotWithMissingDataStream() { assertThat(e.getMessage(), containsString("unable to find data stream [" + missingDataStream + "]")); } + public void testDataStreamAliases() { + Metadata.Builder mdBuilder = Metadata.builder(); + + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-eu"), is(true)); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-us"), is(true)); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-au"), is(true)); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-au"), is(false)); + + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au")); + } + + public void testDataStreamReferToNonExistingDataStream() { + Metadata.Builder mdBuilder = Metadata.builder(); + + Exception e = expectThrows(IllegalArgumentException.class, () -> mdBuilder.put("logs-postgres", "logs-postgres-eu")); + assertThat(e.getMessage(), equalTo("alias [logs-postgres] refers to a non existing data stream [logs-postgres-eu]")); + + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu"); + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + } + + public void testDeleteDataStreamShouldUpdateAlias() { + Metadata.Builder mdBuilder = Metadata.builder(); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au"); + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au")); + + mdBuilder = Metadata.builder(metadata); + mdBuilder.removeDataStream("logs-postgres-us"); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-au")); + + mdBuilder = Metadata.builder(metadata); + mdBuilder.removeDataStream("logs-postgres-au"); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + + mdBuilder = Metadata.builder(metadata); + mdBuilder.removeDataStream("logs-postgres-eu"); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), nullValue()); + } + + public void testDeleteDataStreamAlias() { + Metadata.Builder mdBuilder = Metadata.builder(); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au"); + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au")); + + mdBuilder = Metadata.builder(metadata); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-us", true), is(true)); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-au")); + + mdBuilder = Metadata.builder(metadata); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-au", true), is(true)); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + + mdBuilder = Metadata.builder(metadata); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-eu", true), is(true)); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), nullValue()); + } + + public void testDeleteDataStreamAliasMustExists() { + Metadata.Builder mdBuilder = Metadata.builder(); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au"); + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au")); + + Metadata.Builder mdBuilder2 = Metadata.builder(metadata); + expectThrows(ResourceNotFoundException.class, () -> mdBuilder2.removeDataStreamAlias("logs-mysql", "logs-postgres-us", true)); + assertThat(mdBuilder2.removeDataStreamAlias("logs-mysql", "logs-postgres-us", false), is(false)); + } + public static Metadata randomMetadata() { return randomMetadata(1); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java index d797d98da6f28..11117278b96b2 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.NOT_FOUND; @@ -41,7 +42,7 @@ public void testBareRequest() throws Exception { final AliasMetadata fooAliasMetadata = AliasMetadata.builder("foo").build(); openMapBuilder.put("index", Arrays.asList(fooAliasMetadata, foobarAliasMetadata)); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(false, new String[0], openMapBuilder.build(), - xContentBuilder); + Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{},\"foobar\":{}}}}")); @@ -51,7 +52,7 @@ public void testSimpleAliasWildcardMatchingNothing() throws Exception { final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*" }, openMapBuilder.build(), - xContentBuilder); + Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); @@ -63,7 +64,7 @@ public void testMultipleAliasWildcardsSomeMatching() throws Exception { final AliasMetadata aliasMetadata = AliasMetadata.builder("foobar").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*", "foobar*" }, - openMapBuilder.build(), xContentBuilder); + openMapBuilder.build(), Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foobar\":{}}}}")); @@ -73,7 +74,7 @@ public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foob*", "-foo*" }, - openMapBuilder.build(), xContentBuilder); + openMapBuilder.build(), Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); @@ -85,7 +86,7 @@ public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { final AliasMetadata aliasMetadata = AliasMetadata.builder("foo").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo*", "-foob*" }, - openMapBuilder.build(), xContentBuilder); + openMapBuilder.build(), Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{}}}}")); @@ -104,7 +105,7 @@ public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws E } final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, aliasPattern, openMapBuilder.build(), - xContentBuilder); + Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(NOT_FOUND)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), @@ -115,7 +116,7 @@ public void testAliasWildcardsExcludeExplicitMissing() throws Exception { final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo", "foofoo", "-foo*" }, - openMapBuilder.build(), xContentBuilder); + openMapBuilder.build(), Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index 16deedead18f2..f7a2eaefb5245 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -27,6 +27,7 @@ import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; +import static org.elasticsearch.test.ESTestCase.generateRandomStringArray; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomBoolean; @@ -102,10 +103,18 @@ public static DataStream randomInstance() { return randomInstance(System::currentTimeMillis); } + public static DataStream randomInstance(String name) { + return randomInstance(name, System::currentTimeMillis); + } + public static DataStream randomInstance(LongSupplier timeProvider) { + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + return randomInstance(dataStreamName, timeProvider); + } + + public static DataStream randomInstance(String dataStreamName, LongSupplier timeProvider) { List indices = randomIndexInstances(); long generation = indices.size() + ESTestCase.randomLongBetween(1, 128); - String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(LuceneTestCase.random()))); Map metadata = null; if (randomBoolean()) { @@ -116,6 +125,13 @@ public static DataStream randomInstance(LongSupplier timeProvider) { randomBoolean(), randomBoolean(), false, timeProvider); } + public static DataStreamAlias randomAliasInstance() { + return new DataStreamAlias( + randomAlphaOfLength(5), + List.of(generateRandomStringArray(5, 5, false)) + ); + } + /** * Constructs {@code ClusterState} with the specified data streams and indices. * diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index dba8566d52ab5..24d4034512f26 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -46,6 +46,7 @@ import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtilsForTesting; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -123,6 +124,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -876,6 +878,15 @@ public static List randomList(int minListSize, int maxListSize, Supplier< return list; } + public static Map randomMap(int minListSize, int maxListSize, Supplier> valueConstructor) { + final int size = randomIntBetween(minListSize, maxListSize); + Map list = new HashMap<>(size); + for (int i = 0; i < size; i++) { + Tuple entry = valueConstructor.get(); + list.put(entry.v1(), entry.v2()); + } + return list; + } private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m", "micros", "nanos"}; diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index af3273ec0afc9..9f7611c42b68e 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -18,7 +18,9 @@ import java.util.Collections; import java.util.Map; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class DataStreamsRestIT extends ESRestTestCase { @@ -89,4 +91,108 @@ public void testAddingIndexTemplateWithAliasesAndDataStream() { Exception e = expectThrows(ResponseException.class, () -> client().performRequest(putComposableIndexTemplateRequest)); assertThat(e.getMessage(), containsString("template [my-template] has alias and data stream definitions")); } + + public void testDataStreamAliases() throws Exception { + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\"logs-*\"], \"data_stream\": {}}"); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + Request createDocRequest = new Request("POST", "/logs-myapp1/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + createDocRequest = new Request("POST", "/logs-myapp2/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + // Add logs-myapp1 -> logs & logs-myapp2 -> logs + Request updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity("{\"actions\":[{\"add\":{\"index\":\"logs-myapp1\",\"alias\":\"logs\"}}," + + "{\"add\":{\"index\":\"logs-myapp2\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + Request getAliasesRequest = new Request("GET", "/_aliases"); + Map getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertThat(getAliasesResponse.keySet(), containsInAnyOrder("logs-myapp1", "logs-myapp2")); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + + Request searchRequest = new Request("GET", "/logs/_search"); + Map searchResponse = entityAsMap(client().performRequest(searchRequest)); + assertEquals(2, XContentMapValues.extractValue("hits.total.value", searchResponse)); + + // Remove logs-myapp1 -> logs & logs-myapp2 -> logs + updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity("{\"actions\":[{\"remove\":{\"index\":\"logs-myapp1\",\"alias\":\"logs\"}}," + + "{\"remove\":{\"index\":\"logs-myapp2\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + getAliasesRequest = new Request("GET", "/_aliases"); + getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of(), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); + assertEquals(Map.of(), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/logs/_search"))); + + // Add logs-* -> logs + updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity("{\"actions\":[{\"add\":{\"index\":\"logs-*\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + getAliasesRequest = new Request("GET", "/_aliases"); + getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + + searchRequest = new Request("GET", "/logs/_search"); + searchResponse = entityAsMap(client().performRequest(searchRequest)); + assertEquals(2, XContentMapValues.extractValue("hits.total.value", searchResponse)); + + // Remove logs-* -> logs + updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity("{\"actions\":[{\"remove\":{\"index\":\"logs-*\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + getAliasesRequest = new Request("GET", "/_aliases"); + getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of(), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); + assertEquals(Map.of(), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/logs/_search"))); + } + + public void testDeleteDataStreamApiWithAliasFails() throws IOException { + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\"logs-*\"], \"data_stream\": {}}"); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + Request createDocRequest = new Request("POST", "/logs-emea/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + createDocRequest = new Request("POST", "/logs-nasa/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + Request updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity( + "{\"actions\":[{\"add\":{\"index\":\"logs-emea\",\"alias\":\"logs\"}}," + + "{\"add\":{\"index\":\"logs-nasa\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + Request getAliasesRequest = new Request("GET", "/logs-*/_alias"); + Map getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-emea.aliases", getAliasesResponse)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-nasa.aliases", getAliasesResponse)); + + Exception e = expectThrows(ResponseException.class, () -> client().performRequest(new Request("DELETE", "/_data_stream/logs"))); + assertThat(e.getMessage(), containsString("The provided expression [logs] matches an alias, " + + "specify the corresponding concrete indices instead")); + + assertOK(client().performRequest(new Request("DELETE", "/_data_stream/logs-emea"))); + assertOK(client().performRequest(new Request("DELETE", "/_data_stream/logs-nasa"))); + + getAliasesRequest = new Request("GET", "/logs-*/_alias"); + assertThat(entityAsMap(client().performRequest(getAliasesRequest)), equalTo(Map.of())); + } } diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 88cfa86b91eb4..1c4f03e384a72 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; @@ -42,6 +44,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Nullable; @@ -566,7 +569,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(dataStreamName, client().admin().indices().prepareUpgrade(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareRecoveries(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareUpgradeStatus(dataStreamName), false); - verifyResolvability(dataStreamName, client().admin().indices().prepareGetAliases("dummy").addIndices(dataStreamName), true); + verifyResolvability(dataStreamName, client().admin().indices().prepareGetAliases("dummy").addIndices(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareGetFieldMappings(dataStreamName), false); verifyResolvability( dataStreamName, @@ -685,7 +688,7 @@ public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exceptio assertTrue(maybeE.isPresent()); } - public void testAliasActionsFailOnDataStreams() throws Exception { + public void testAliasActionsOnDataStreams() throws Exception { putComposableIndexTemplate("id1", List.of("metrics-foo*")); String dataStreamName = "metrics-foo"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); @@ -696,8 +699,12 @@ public void testAliasActionsFailOnDataStreams() throws Exception { .aliases("foo"); IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest(); aliasesAddRequest.addAliasAction(addAction); - Exception e = expectThrows(IndexNotFoundException.class, () -> client().admin().indices().aliases(aliasesAddRequest).actionGet()); - assertThat(e.getMessage(), equalTo("no such index [" + dataStreamName + "]")); + assertAcked(client().admin().indices().aliases(aliasesAddRequest).actionGet()); + GetAliasesResponse response = client().admin().indices().getAliases(new GetAliasesRequest()).actionGet(); + assertThat( + response.getDataStreamAliases(), + equalTo(Map.of("metrics-foo", List.of(new DataStreamAlias("foo", List.of("metrics-foo"))))) + ); } public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 61f17d8063300..b624c5cdcbf09 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -1648,18 +1648,18 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithWildcard() { final User user = new User("data-stream-tester2", "data_stream_test2"); GetAliasesRequest request = new GetAliasesRequest("*"); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams and their backing indices should _not_ be in the authorized list since the backing indices // do not match the requested pattern List dataStreams = org.elasticsearch.common.collect.List.of("logs-foo", "logs-foobar"); final List authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request); for (String dsName : dataStreams) { - assertThat(authorizedIndices, not(hasItem(dsName))); + assertThat(authorizedIndices, hasItem(dsName)); DataStream dataStream = metadata.dataStreams().get(dsName); - assertThat(authorizedIndices, not(hasItem(dsName))); + assertThat(authorizedIndices, hasItem(dsName)); for (Index i : dataStream.getIndices()) { - assertThat(authorizedIndices, not(hasItem(i.getName()))); + assertThat(authorizedIndices, hasItem(i.getName())); } } @@ -1667,11 +1667,11 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithWildcard() { // pattern ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices); for (String dsName : dataStreams) { - assertThat(resolvedIndices.getLocal(), not(hasItem(dsName))); + assertThat(resolvedIndices.getLocal(), hasItem(dsName)); DataStream dataStream = metadata.dataStreams().get(dsName); - assertThat(resolvedIndices.getLocal(), not(hasItem(dsName))); + assertThat(resolvedIndices.getLocal(), hasItem(dsName)); for (Index i : dataStream.getIndices()) { - assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName()))); + assertThat(resolvedIndices.getLocal(), hasItem(i.getName())); } } } @@ -1681,24 +1681,24 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithoutWildcard( String dataStreamName = "logs-foobar"; GetAliasesRequest request = new GetAliasesRequest(dataStreamName); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams and their backing indices should _not_ be in the authorized list since the backing indices // do not match the requested name final List authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request); - assertThat(authorizedIndices, not(hasItem(dataStreamName))); + assertThat(authorizedIndices, hasItem(dataStreamName)); DataStream dataStream = metadata.dataStreams().get(dataStreamName); - assertThat(authorizedIndices, not(hasItem(dataStreamName))); + assertThat(authorizedIndices, hasItem(dataStreamName)); for (Index i : dataStream.getIndices()) { - assertThat(authorizedIndices, not(hasItem(i.getName()))); + assertThat(authorizedIndices, hasItem(i.getName())); } // neither data streams nor their backing indices will be in the resolved list since the backing indices do not match the // requested name(s) ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices); - assertThat(resolvedIndices.getLocal(), not(hasItem(dataStreamName))); + assertThat(resolvedIndices.getLocal(), hasItem(dataStreamName)); for (Index i : dataStream.getIndices()) { - assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName()))); + assertThat(resolvedIndices.getLocal(), hasItem(i.getName())); } } @@ -1795,24 +1795,24 @@ public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildca String dataStreamName = "logs-foobar"; GetAliasesRequest request = new GetAliasesRequest(dataStreamName); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams and their backing indices should _not_ be in the authorized list since the backing indices // did not match the requested pattern and the request does not support data streams final List authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request); - assertThat(authorizedIndices, not(hasItem(dataStreamName))); + assertThat(authorizedIndices, hasItem(dataStreamName)); DataStream dataStream = metadata.dataStreams().get(dataStreamName); - assertThat(authorizedIndices, not(hasItem(dataStreamName))); + assertThat(authorizedIndices, hasItem(dataStreamName)); for (Index i : dataStream.getIndices()) { - assertThat(authorizedIndices, not(hasItem(i.getName()))); + assertThat(authorizedIndices, hasItem(i.getName())); } // neither data streams nor their backing indices will be in the resolved list since the request does not support data streams // and the backing indices do not match the requested name ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices); - assertThat(resolvedIndices.getLocal(), not(hasItem(dataStreamName))); + assertThat(resolvedIndices.getLocal(), hasItem(dataStreamName)); for (Index i : dataStream.getIndices()) { - assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName()))); + assertThat(resolvedIndices.getLocal(), hasItem(i.getName())); } } @@ -1867,7 +1867,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar String indexName = ".ds-logs-foobar-*"; GetAliasesRequest request = new GetAliasesRequest(indexName); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams should _not_ be in the authorized list but their backing indices that matched both the requested pattern // and the authorized pattern should be in the list @@ -1893,7 +1893,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAnd String indexName = ".ds-logs-foobar-*"; GetAliasesRequest request = new GetAliasesRequest(indexName); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams should _not_ be in the authorized list but a single backing index that matched the requested pattern // and the authorized name should be in the list diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml new file mode 100644 index 0000000000000..0050ee20f382c --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml @@ -0,0 +1,73 @@ +--- +"Create data stream alias": + - skip: + version: " - 7.99.99" + reason: "data streams alias not yet backported to the 7.x branch" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [events-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [events-*] + template: + settings: + index.number_of_replicas: 0 + data_stream: {} + + - do: + indices.create_data_stream: + name: events-app1 + - is_true: acknowledged + + - do: + index: + index: events-app1 + refresh: true + body: + '@timestamp': '2022-12-12' + foo: bar + + - do: + indices.create_data_stream: + name: events-app2 + - is_true: acknowledged + + - do: + index: + index: events-app2 + refresh: true + body: + '@timestamp': '2022-12-12' + foo: bar + + - do: + indices.update_aliases: + body: + actions: + - add: + index: events-app1 + alias: events + - add: + index: events-app2 + alias: events + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: events-app1 } + - match: { data_streams.1.name: events-app2 } + + - do: + indices.get_alias: {} + + - match: {events-app1.aliases.events: {}} + - match: {events-app2.aliases.events: {}} + + - do: + search: + index: events + body: { query: { match_all: {} } } + - length: { hits.hits: 2 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/50_data_streams.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/50_data_streams.yml index e5afc18095be4..c832ff9818a26 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/50_data_streams.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/50_data_streams.yml @@ -192,8 +192,8 @@ teardown: - do: headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user - indices.get_alias: - name: easy* + indices.get: + index: easy* - match: {easy-index.aliases.easy-alias: {}} - is_false: easy-data-stream1 diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/51_data_stream_aliases.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/51_data_stream_aliases.yml new file mode 100644 index 0000000000000..ac506b19ccddf --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/51_data_stream_aliases.yml @@ -0,0 +1,286 @@ +--- +setup: + - skip: + features: ["headers", "allowed_warnings"] + version: " - 7.99.99" + reason: "data stream aliases not yet backported to 7.x branch" + + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_role: + name: "ingest_events_role" + body: > + { + "indices": [ + { "names": ["events*"], "privileges": ["create_doc", "create_index"] } + ] + } + + - do: + security.put_role: + name: "query_events_role" + body: > + { + "indices": [ + { "names": ["app1", "events*"], "privileges": ["read", "view_index_metadata", "monitor"] } + ] + } + + - do: + security.put_role: + name: "admin_events_role" + body: > + { + "indices": [ + { "names": ["events*"], "privileges": ["read", "write", "manage"] } + ] + } + + - do: + security.put_role: + name: "ingest_logs_role" + body: > + { + "indices": [ + { "names": ["logs*"], "privileges": ["create_doc", "create_index"] } + ] + } + + - do: + security.put_role: + name: "query_logs_role" + body: > + { + "indices": [ + { "names": ["app1", "logs*"], "privileges": ["read", "view_index_metadata", "monitor"] } + ] + } + + - do: + security.put_role: + name: "admin_logs_role" + body: > + { + "indices": [ + { "names": ["logs*"], "privileges": ["read", "write", "manage"] } + ] + } + + - do: + security.put_user: + username: "test_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "ingest_events_role", "query_events_role" ], + "full_name" : "user with privileges on event data streams" + } + + - do: + security.put_user: + username: "no_authz_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "ingest_logs_role", "query_logs_role" ], + "full_name" : "user with privileges on logs data streams" + } + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [events-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [events-*] + template: + mappings: + properties: + '@timestamp': + type: date + 'foo': + type: keyword + data_stream: {} + + - do: + allowed_warnings: + - "index template [my-template2] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [logs-*] + template: + mappings: + properties: + '@timestamp': + type: date + 'foo': + type: keyword + data_stream: {} + +--- +teardown: + - do: + security.delete_user: + username: "test_user" + ignore: 404 + + - do: + security.delete_user: + username: "test_user2" + ignore: 404 + + - do: + security.delete_role: + name: "ingest_events_role" + ignore: 404 + + - do: + security.delete_role: + name: "query_events_role" + ignore: 404 + + - do: + security.delete_role: + name: "admin_events_role" + ignore: 404 + + - do: + security.delete_role: + name: "ingest_logs_role" + ignore: 404 + + - do: + security.delete_role: + name: "query_logs_role" + ignore: 404 + + - do: + security.delete_role: + name: "admin_logs_role" + ignore: 404 + +--- +"Basic read authorization test": + - skip: + features: ["headers", "allowed_warnings"] + version: " - 7.99.99" + reason: "data stream aliases not yet backported to 7.x branch" + + - do: + index: + index: events-app1 + refresh: true + body: + '@timestamp': '2022-12-12' + foo: bar + + - do: + index: + index: logs-app1 + refresh: true + body: + '@timestamp': '2022-12-12' + bar: baz + + - do: + indices.update_aliases: + body: + actions: + - add: + index: events-app1 + alias: app1 + - add: + index: logs-app1 + alias: app1 + + # app1 allows access to both data streams for this user irregardless whether privileges have been granted on data streams + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: app1 + - match: { hits.total: 2 } + - match: { hits.hits.0._source.foo: bar } + - match: { hits.hits.1._source.bar: baz } + + # this user is authorized to use events-app1 data stream directly + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: events-app1 + - match: { hits.total: 1 } + - match: { hits.hits.0._source.foo: bar } + + # this user is authorized to use a backing index of events-app1 data stream directly + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: .ds-events-app1* + - match: { hits.total: 1 } + - match: { hits.hits.0._source.foo: bar } + + # this user isn't authorized to use logs-app1 data stream directly + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + catch: forbidden + search: + rest_total_hits_as_int: true + index: logs-app1 + + # this user isn't authorized to use a backing index of logs-app1 data stream directly + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: .ds-logs-app1* + - match: { hits.total: 0 } + + # app1 allows access to both data streams for this user irregardless whether privileges have been granted on data streams + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + search: + rest_total_hits_as_int: true + index: app1 + - match: { hits.total: 2 } + - match: { hits.hits.0._source.foo: bar } + - match: { hits.hits.1._source.bar: baz } + + # this user isn't authorized to use events-app1 data stream directly + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + catch: forbidden + search: + rest_total_hits_as_int: true + index: events-app1 + + # this user isn't authorized to use a backing index of events-app1 data stream directly + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + search: + rest_total_hits_as_int: true + index: .ds-events-app1* + - match: { hits.total: 0 } + + # this user is authorized to use logs-app1 data stream directly + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + search: + rest_total_hits_as_int: true + index: logs-app1 + - match: { hits.total: 1 } + - match: { hits.hits.0._source.bar: baz } + + # this user is authorized to use a backing index of logs-app1 data stream directly + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + search: + rest_total_hits_as_int: true + index: .ds-logs-app1* + - match: { hits.total: 1 } + - match: { hits.hits.0._source.bar: baz }