From eac3afff46e61a873d7de0d1dae69671ffbdeec7 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 11 May 2021 09:51:05 +0200 Subject: [PATCH] Add basic alias support for data streams Backporting #72613 to 7.x. Aliases to data streams can be defined via the existing update aliases api. Aliases can either only refer to data streams or to indices (not both). Also the existing get aliases api has been modified to support returning aliases that refer to data streams. Aliases for data streams are stored separately from data streams and and refer to data streams by name and not to the backing indices of a data stream. This means that when backing indices are added or removed from a data stream that then the data stream alias doesn't need to be updated. The authorization model for aliases that refer to data streams is the same as for aliases the refer to indices. In security privileges can be defined on aliases, indices and data streams. When a privilege is granted on an alias then access is also granted on the indices that an alias refers to (irregardless whether privileges are granted or denied on the actual indices). The same will apply for aliases that refer to data streams. See for more details: https://github.com/elastic/elasticsearch/issues/66163#issuecomment-824709767 Relates to #66163 --- docs/reference/indices/aliases.asciidoc | 98 ++++++ .../template/SimpleIndexTemplateIT.java | 2 +- .../alias/TransportIndicesAliasesAction.java | 21 ++ .../indices/alias/get/GetAliasesRequest.java | 5 + .../indices/alias/get/GetAliasesResponse.java | 21 +- .../alias/get/TransportGetAliasesAction.java | 31 +- .../indices/resolve/ResolveIndexAction.java | 5 +- .../cluster/metadata/AliasAction.java | 56 ++++ .../cluster/metadata/AliasValidator.java | 19 +- .../cluster/metadata/DataStreamAlias.java | 102 +++++++ .../cluster/metadata/DataStreamMetadata.java | 62 +++- .../cluster/metadata/IndexAbstraction.java | 51 ++++ .../metadata/IndexNameExpressionResolver.java | 9 +- .../cluster/metadata/Metadata.java | 157 ++++++++-- .../metadata/MetadataIndexAliasesService.java | 32 +- .../admin/indices/RestGetAliasesAction.java | 22 +- .../indices/alias/AliasActionsTests.java | 4 +- .../alias/get/GetAliasesResponseTests.java | 8 +- .../get/TransportGetAliasesActionTests.java | 13 +- .../ComposableIndexTemplateTests.java | 2 +- .../metadata/DataStreamMetadataTests.java | 21 +- .../metadata/DataStreamTemplateTests.java | 38 +++ .../MetadataIndexAliasesServiceTests.java | 28 ++ .../cluster/metadata/MetadataTests.java | 117 ++++++- .../indices/RestGetAliasesActionTests.java | 15 +- .../metadata/DataStreamTestHelper.java | 18 +- .../org/elasticsearch/test/ESTestCase.java | 11 + .../xpack/datastreams/DataStreamsRestIT.java | 106 +++++++ .../datastreams/DataStreamIT.java | 15 +- .../authz/IndicesAndAliasesResolverTests.java | 42 +-- .../data_stream/140_data_stream_aliases.yml | 73 +++++ .../test/security/authz/50_data_streams.yml | 4 +- .../security/authz/51_data_stream_aliases.yml | 286 ++++++++++++++++++ 33 files changed, 1399 insertions(+), 95 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java create mode 100644 x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml create mode 100644 x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/51_data_stream_aliases.yml diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index d64edeed9996f..817adad281ed9 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -511,3 +511,101 @@ POST /_aliases } -------------------------------------------------- // TEST[s/^/PUT test\nPUT test2\n/] + +[[aliases-data-streams]] +===== Data stream aliases + +An alias can also point to one or more data streams. To add a data stream to an +alias, specify the stream's name in the `index` parameter. Wildcards (`*`) are +supported. If a wildcard pattern matches both data streams and indices, the +action only uses the matching data streams. + +You can only specify data streams in the `add` and `remove` actions. Aliases +that point to data streams do not support the following parameters: + +* `filter` +* `index_routing` +* `is_write_index` +* `routing` +* `search_routing` + +For example, the following request adds two data streams to the `logs` alias. + +//// +[source,console] +---- +PUT _data_stream/logs-my_app-default + +PUT _data_stream/logs-nginx.access-prod +---- +//// + +[source,console] +---- +POST _aliases +{ + "actions": [ + { "add": { "index": "logs-my_app-default", "alias": "logs" }}, + { "add": { "index": "logs-nginx.access-prod", "alias": "logs" }} + ] +} +---- +// TEST[continued] + +To verify the alias points to both data streams, use the +<>. + +[source,console] +---- +GET logs-*/_alias +---- +// TEST[continued] + +The API returns: + +[source,console-result] +---- +{ + "logs-my_app-default": { + "aliases": { + "logs": {} + } + }, + "logs-nginx.access-prod": { + "aliases": { + "logs": {} + } + } +} +---- + +Use the `remove` action to remove a data stream from an alias. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { "remove": { "index": "logs-my_app-default", "alias": "logs" }} + ] +} + +GET logs-*/_alias +---- +// TEST[continued] + +The get index alias API returns: + +[source,console-result] +---- +{ + "logs-my_app-default": { + "aliases": {} + }, + "logs-nginx.access-prod": { + "aliases": { + "logs": {} + } + } +} +---- diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index a588dc9ebb638..a673a2587c5cf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -548,7 +548,7 @@ public void testAliasNameExistingIndex() throws Exception { InvalidAliasNameException e = expectThrows(InvalidAliasNameException.class, () -> createIndex("test")); - assertThat(e.getMessage(), equalTo("Invalid alias name [index], an index exists with the same name as the alias")); + assertThat(e.getMessage(), equalTo("Invalid alias name [index]: an index or data stream exists with the same name as the alias")); } public void testAliasEmptyName() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 5f428cf9af1ab..4e63aa4496260 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -88,6 +88,27 @@ protected void masterOperation(final IndicesAliasesRequest request, final Cluste // Resolve all the AliasActions into AliasAction instances and gather all the aliases Set aliases = new HashSet<>(); for (AliasActions action : actions) { + List concreteDataStreams = + indexNameExpressionResolver.dataStreamNames(state, request.indicesOptions(), action.indices()); + if (concreteDataStreams.size() != 0) { + switch (action.actionType()) { + case ADD: + for (String dataStreamName : concreteDataStreams) { + finalActions.add(new AliasAction.AddDataStreamAlias(action.aliases()[0], dataStreamName)); + } + break; + case REMOVE: + for (String dataStreamName : concreteDataStreams) { + finalActions.add( + new AliasAction.RemoveDataStreamAlias(action.aliases()[0], dataStreamName, action.mustExist())); + } + break; + default: + throw new IllegalArgumentException("Unsupported action [" + action.actionType() + "]"); + } + continue; + } + final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request.indicesOptions(), false, action.indices()); for (Index concreteIndex : concreteIndices) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java index 2cfffd2699d6e..009e3a9435649 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java @@ -107,4 +107,9 @@ public IndicesOptions indicesOptions() { public ActionRequestValidationException validate() { return null; } + + @Override + public boolean includeDataStreams() { + return true; + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java index f4ba1580fb8bc..a98a4d21132cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java @@ -10,34 +10,48 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.DataStreamAlias; +import org.elasticsearch.cluster.metadata.DataStreamMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Objects; public class GetAliasesResponse extends ActionResponse { private final ImmutableOpenMap> aliases; + private final Map> dataStreamAliases; - public GetAliasesResponse(ImmutableOpenMap> aliases) { + public GetAliasesResponse(ImmutableOpenMap> aliases, Map> dataStreamAliases) { this.aliases = aliases; + this.dataStreamAliases = dataStreamAliases; } public GetAliasesResponse(StreamInput in) throws IOException { super(in); aliases = in.readImmutableMap(StreamInput::readString, i -> i.readList(AliasMetadata::new)); + dataStreamAliases = in.getVersion().onOrAfter(DataStreamMetadata.DATA_STREAM_ALIAS_VERSION) ? + in.readMap(StreamInput::readString, in1 -> in1.readList(DataStreamAlias::new)) : Map.of(); } public ImmutableOpenMap> getAliases() { return aliases; } + public Map> getDataStreamAliases() { + return dataStreamAliases; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(aliases, StreamOutput::writeString, StreamOutput::writeList); + if (out.getVersion().onOrAfter(DataStreamMetadata.DATA_STREAM_ALIAS_VERSION)) { + out.writeMap(dataStreamAliases, StreamOutput::writeString, StreamOutput::writeList); + } } @Override @@ -49,11 +63,12 @@ public boolean equals(Object o) { return false; } GetAliasesResponse that = (GetAliasesResponse) o; - return Objects.equals(aliases, that.aliases); + return Objects.equals(aliases, that.aliases) && + Objects.equals(dataStreamAliases, that.dataStreamAliases); } @Override public int hashCode() { - return Objects.hash(aliases); + return Objects.hash(aliases, dataStreamAliases); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 216241fbde980..e4273fa032012 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -14,6 +14,8 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.DataStreamAlias; +import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -21,6 +23,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; @@ -30,8 +33,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -67,7 +72,7 @@ protected void masterOperation(GetAliasesRequest request, ClusterState state, Ac final SystemIndexAccessLevel systemIndexAccessLevel = indexNameExpressionResolver.getSystemIndexAccessLevel(); ImmutableOpenMap> aliases = state.metadata().findAliases(request, concreteIndices); listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases, state, - systemIndexAccessLevel, threadPool.getThreadContext(), systemIndices))); + systemIndexAccessLevel, threadPool.getThreadContext(), systemIndices), postProcess(request, state))); } /** @@ -80,6 +85,15 @@ static ImmutableOpenMap> postProcess(GetAliasesReque boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0; ImmutableOpenMap.Builder> mapBuilder = ImmutableOpenMap.builder(aliases); for (String index : concreteIndices) { + IndexAbstraction ia = state.metadata().getIndicesLookup().get(index); + assert ia.getType() == IndexAbstraction.Type.CONCRETE_INDEX; + if (ia.getParentDataStream() != null) { + // Don't include backing indices of data streams, + // because it is just noise. Aliases can't refer + // to backing indices directly. + continue; + } + if (aliases.get(index) == null && noAliasesSpecified) { List previous = mapBuilder.put(index, Collections.emptyList()); assert previous == null; @@ -92,6 +106,21 @@ static ImmutableOpenMap> postProcess(GetAliasesReque return finalResponse; } + Map> postProcess(GetAliasesRequest request, ClusterState state) { + Map> result = new HashMap<>(); + boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0; + List requestedDataStreams = + indexNameExpressionResolver.dataStreamNames(state, request.indicesOptions(), request.indices()); + for (String requestedDataStream : requestedDataStreams) { + List aliases = state.metadata().dataStreamAliases().values().stream() + .filter(alias -> alias.getDataStreams().contains(requestedDataStream)) + .filter(alias -> noAliasesSpecified || Regex.simpleMatch(request.aliases(), alias.getName())) + .collect(Collectors.toList()); + result.put(requestedDataStream, aliases); + } + return result; + } + private static void checkSystemIndexAccess(GetAliasesRequest request, SystemIndices systemIndices, ClusterState state, ImmutableOpenMap> aliasesMap, SystemIndexAccessLevel systemIndexAccessLevel, ThreadContext threadContext) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 46bd1822587e1..3b70fd628ac26 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -560,10 +560,9 @@ private static void enrichIndexAbstraction(String indexAbstraction, SortedMap i.getIndex().getName()).toArray(String[]::new); + String[] indexNames = ia.getIndices().stream().map(i -> i.getIndex().getName()).toArray(String[]::new); Arrays.sort(indexNames); - aliases.add(new ResolvedAlias(alias.getName(), indexNames)); + aliases.add(new ResolvedAlias(ia.getName(), indexNames)); break; case DATA_STREAM: IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) ia; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java index a837a7672b3b6..426e6bd393b71 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java @@ -198,4 +198,60 @@ boolean apply(NewAliasValidator aliasValidator, Metadata.Builder metadata, Index throw new UnsupportedOperationException(); } } + + public static class AddDataStreamAlias extends AliasAction { + + private final String aliasName; + private final String dataStreamName; + + public AddDataStreamAlias(String aliasName, String dataStreamName) { + super(dataStreamName); + this.aliasName = aliasName; + this.dataStreamName = dataStreamName; + } + + public String getAliasName() { + return aliasName; + } + + public String getDataStreamName() { + return dataStreamName; + } + + @Override + boolean removeIndex() { + return false; + } + + @Override + boolean apply(NewAliasValidator aliasValidator, Metadata.Builder metadata, IndexMetadata index) { + aliasValidator.validate(aliasName, null, null, null); + return metadata.put(aliasName, dataStreamName); + } + } + + public static class RemoveDataStreamAlias extends AliasAction { + + private final String aliasName; + private final Boolean mustExist; + private final String dataStreamName; + + public RemoveDataStreamAlias(String aliasName, String dataStreamName, Boolean mustExist) { + super(dataStreamName); + this.aliasName = aliasName; + this.mustExist = mustExist; + this.dataStreamName = dataStreamName; + } + + @Override + boolean removeIndex() { + return false; + } + + @Override + boolean apply(NewAliasValidator aliasValidator, Metadata.Builder metadata, IndexMetadata index) { + boolean mustExist = this.mustExist != null ? this.mustExist : false; + return metadata.removeDataStreamAlias(aliasName, dataStreamName, mustExist); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java index cb8e3aaa7eac1..cf7a091a8b751 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Optional; import java.util.function.Function; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -39,7 +40,7 @@ public class AliasValidator { * @throws IllegalArgumentException if the alias is not valid */ public void validateAlias(Alias alias, String index, Metadata metadata) { - validateAlias(alias.name(), index, alias.indexRouting(), metadata::index); + validateAlias(alias.name(), index, alias.indexRouting(), lookup(metadata)); } /** @@ -48,7 +49,7 @@ public void validateAlias(Alias alias, String index, Metadata metadata) { * @throws IllegalArgumentException if the alias is not valid */ public void validateAliasMetadata(AliasMetadata aliasMetadata, String index, Metadata metadata) { - validateAlias(aliasMetadata.alias(), index, aliasMetadata.indexRouting(), metadata::index); + validateAlias(aliasMetadata.alias(), index, aliasMetadata.indexRouting(), lookup(metadata)); } /** @@ -72,16 +73,16 @@ public void validateAliasStandalone(Alias alias) { /** * Validate a proposed alias. */ - public void validateAlias(String alias, String index, @Nullable String indexRouting, Function indexLookup) { + public void validateAlias(String alias, String index, @Nullable String indexRouting, Function lookup) { validateAliasStandalone(alias, indexRouting); if (Strings.hasText(index) == false) { throw new IllegalArgumentException("index name is required"); } - IndexMetadata indexNamedSameAsAlias = indexLookup.apply(alias); - if (indexNamedSameAsAlias != null) { - throw new InvalidAliasNameException(indexNamedSameAsAlias.getIndex(), alias, "an index exists with the same name as the alias"); + String sameNameAsAlias = lookup.apply(alias); + if (sameNameAsAlias != null) { + throw new InvalidAliasNameException(alias, "an index or data stream exists with the same name as the alias"); } } @@ -134,4 +135,10 @@ private static void validateAliasFilter(XContentParser parser, SearchExecutionCo QueryBuilder queryBuilder = Rewriteable.rewrite(parseInnerQueryBuilder, searchExecutionContext, true); queryBuilder.toQuery(searchExecutionContext); } + + private static Function lookup(Metadata metadata) { + return name -> Optional.ofNullable(metadata.getIndicesLookup().get(name)) + .filter(indexAbstraction -> indexAbstraction.getType() != IndexAbstraction.Type.ALIAS) + .map(IndexAbstraction::getName).orElse(null); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java new file mode 100644 index 0000000000000..9bf8367724ea5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class DataStreamAlias extends AbstractDiffable implements ToXContentObject { + + public static final ParseField DATA_STREAMS_FIELD = new ParseField("data_streams"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "data_stream_alias", + false, + (args, name) -> new DataStreamAlias(name, (List) args[0]) + ); + + static { + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), DATA_STREAMS_FIELD); + } + + private final String name; + private final List dataStreams; + + public DataStreamAlias(String name, List dataStreams) { + this.name = Objects.requireNonNull(name); + this.dataStreams = List.copyOf(dataStreams); + } + + public DataStreamAlias(StreamInput in) throws IOException { + this.name = in.readString(); + this.dataStreams = in.readStringList(); + } + + public String getName() { + return name; + } + + public List getDataStreams() { + return dataStreams; + } + + public static Diff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(DataStreamAlias::new, in); + } + + public static DataStreamAlias fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + if (token != XContentParser.Token.FIELD_NAME) { + throw new ParsingException(parser.getTokenLocation(), "unexpected token"); + } + String name = parser.currentName(); + DataStreamAlias alias = PARSER.parse(parser, name); + return alias; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name); + builder.field(DATA_STREAMS_FIELD.getPreferredName(), dataStreams); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeStringCollection(dataStreams); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DataStreamAlias that = (DataStreamAlias) o; + return Objects.equals(name, that.name) && + Objects.equals(dataStreams, that.dataStreams); + } + + @Override + public int hashCode() { + return Objects.hash(name, dataStreams); + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java index f3164a6bc5219..278d3d46eb3be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java @@ -34,9 +34,17 @@ public class DataStreamMetadata implements Metadata.Custom { public static final String TYPE = "data_stream"; private static final ParseField DATA_STREAM = new ParseField("data_stream"); + private static final ParseField DATA_STREAM_ALIASES = new ParseField("data_stream_aliases"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(TYPE, false, - a -> new DataStreamMetadata((Map) a[0])); + args -> { + Map dataStreams = (Map) args[0]; + Map dataStreamAliases = (Map) args[1]; + if (dataStreamAliases == null) { + dataStreamAliases = Map.of(); + } + return new DataStreamMetadata(dataStreams, dataStreamAliases); + }); static { PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { @@ -47,22 +55,40 @@ public class DataStreamMetadata implements Metadata.Custom { } return dataStreams; }, DATA_STREAM); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + Map dataStreams = new HashMap<>(); + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + DataStreamAlias alias = DataStreamAlias.fromXContent(p); + dataStreams.put(alias.getName(), alias); + } + return dataStreams; + }, DATA_STREAM_ALIASES); } + public static final Version DATA_STREAM_ALIAS_VERSION = Version.V_8_0_0; + private final Map dataStreams; + private final Map dataStreamAliases; - public DataStreamMetadata(Map dataStreams) { + public DataStreamMetadata(Map dataStreams, + Map dataStreamAliases) { this.dataStreams = Collections.unmodifiableMap(new HashMap<>(dataStreams)); + this.dataStreamAliases = Map.copyOf(dataStreamAliases); } public DataStreamMetadata(StreamInput in) throws IOException { - this(in.readMap(StreamInput::readString, DataStream::new)); + this(in.readMap(StreamInput::readString, DataStream::new), in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION) ? + in.readMap(StreamInput::readString, DataStreamAlias::new) : Map.of()); } public Map dataStreams() { return this.dataStreams; } + public Map getDataStreamAliases() { + return dataStreamAliases; + } + @Override public Diff diff(Metadata.Custom before) { return new DataStreamMetadata.DataStreamMetadataDiff((DataStreamMetadata) before, this); @@ -90,6 +116,9 @@ public Version getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(this.dataStreams, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + out.writeMap(this.dataStreamAliases, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + } } public static DataStreamMetadata fromXContent(XContentParser parser) throws IOException { @@ -103,12 +132,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(dataStream.getKey(), dataStream.getValue()); } builder.endObject(); + builder.startObject(DATA_STREAM_ALIASES.getPreferredName()); + for (Map.Entry dataStream : dataStreamAliases.entrySet()) { + dataStream.getValue().toXContent(builder, params); + } + builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(this.dataStreams); + return Objects.hash(this.dataStreams, dataStreamAliases); } @Override @@ -120,7 +154,8 @@ public boolean equals(Object obj) { return false; } DataStreamMetadata other = (DataStreamMetadata) obj; - return Objects.equals(this.dataStreams, other.dataStreams); + return Objects.equals(this.dataStreams, other.dataStreams) && + Objects.equals(this.dataStreamAliases, other.dataStreamAliases); } @Override @@ -131,25 +166,40 @@ public String toString() { static class DataStreamMetadataDiff implements NamedDiff { final Diff> dataStreamDiff; + final Diff> dataStreamAliasDiff; DataStreamMetadataDiff(DataStreamMetadata before, DataStreamMetadata after) { this.dataStreamDiff = DiffableUtils.diff(before.dataStreams, after.dataStreams, DiffableUtils.getStringKeySerializer()); + this.dataStreamAliasDiff = DiffableUtils.diff(before.dataStreamAliases, after.dataStreamAliases, + DiffableUtils.getStringKeySerializer()); } DataStreamMetadataDiff(StreamInput in) throws IOException { this.dataStreamDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DataStream::new, DataStream::readDiffFrom); + if (in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + this.dataStreamAliasDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), + DataStreamAlias::new, DataStreamAlias::readDiffFrom); + } else { + this.dataStreamAliasDiff = null; + } } @Override public Metadata.Custom apply(Metadata.Custom part) { - return new DataStreamMetadata(dataStreamDiff.apply(((DataStreamMetadata) part).dataStreams)); + return new DataStreamMetadata( + dataStreamDiff.apply(((DataStreamMetadata) part).dataStreams), + dataStreamAliasDiff != null ? dataStreamAliasDiff.apply(((DataStreamMetadata) part).dataStreamAliases) : Map.of() + ); } @Override public void writeTo(StreamOutput out) throws IOException { dataStreamDiff.writeTo(out); + if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + dataStreamAliasDiff.writeTo(out); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index f1f71b62375b2..91c90d6d2b6c8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -303,4 +303,55 @@ public org.elasticsearch.cluster.metadata.DataStream getDataStream() { return dataStream; } } + + class DataStreamAlias implements IndexAbstraction { + + private final org.elasticsearch.cluster.metadata.DataStreamAlias dataStreamAlias; + private final List indicesOfAllDataStreams; + + public DataStreamAlias(org.elasticsearch.cluster.metadata.DataStreamAlias dataStreamAlias, + List indicesOfAllDataStreams) { + this.dataStreamAlias = dataStreamAlias; + this.indicesOfAllDataStreams = indicesOfAllDataStreams; + } + + @Override + public Type getType() { + return Type.ALIAS; + } + + @Override + public String getName() { + return dataStreamAlias.getName(); + } + + @Override + public List getIndices() { + return indicesOfAllDataStreams; + } + + @Override + public IndexMetadata getWriteIndex() { + return null; + } + + @Override + public DataStream getParentDataStream() { + return null; + } + + @Override + public boolean isHidden() { + return false; + } + + @Override + public boolean isSystem() { + return false; + } + + public org.elasticsearch.cluster.metadata.DataStreamAlias getDataStreamAlias() { + return dataStreamAlias; + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 73acca17b1b5c..bebb87698e834 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -141,8 +141,11 @@ public List dataStreamNames(ClusterState state, IndicesOptions options, indexExpressions = new String[]{"*"}; } - List dataStreams = wildcardExpressionResolver.resolve(context, Arrays.asList(indexExpressions)); - return ((dataStreams == null) ? org.elasticsearch.common.collect.List.of() : dataStreams).stream() + List expressions = Arrays.asList(indexExpressions); + for (ExpressionResolver expressionResolver : expressionResolvers) { + expressions = expressionResolver.resolve(context, expressions); + } + return ((expressions == null) ? org.elasticsearch.common.collect.List.of() : expressions).stream() .map(x -> state.metadata().getIndicesLookup().get(x)) .filter(Objects::nonNull) .filter(ia -> ia.getType() == IndexAbstraction.Type.DATA_STREAM) @@ -594,7 +597,7 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab String concreteIndex = index.getIndex().getName(); AliasMetadata aliasMetadata = index.getAliases().get(indexAbstraction.getName()); if (norouting.contains(concreteIndex) == false) { - if (aliasMetadata.searchRoutingValues().isEmpty() == false) { + if (aliasMetadata != null && aliasMetadata.searchRoutingValues().isEmpty() == false) { // Routing alias if (routings == null) { routings = new HashMap<>(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 5b028a8bdf0f0..830260478ebc2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -15,6 +15,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.cluster.ClusterState; @@ -277,6 +278,19 @@ public boolean equalsAliases(Metadata other) { } } + if (other.dataStreamAliases().size() != dataStreamAliases().size()) { + return false; + } + for (DataStreamAlias otherAlias : other.dataStreamAliases().values()) { + DataStreamAlias thisAlias = dataStreamAliases().get(otherAlias.getName()); + if (thisAlias == null) { + return false; + } + if (thisAlias.equals(otherAlias) == false) { + return false; + } + } + return true; } @@ -611,22 +625,12 @@ public String resolveWriteIndexRouting(@Nullable String routing, String aliasOrI if (writeIndex == null) { throw new IllegalArgumentException("alias [" + aliasOrIndex + "] does not have a write index"); } - AliasMetadata aliasMd = writeIndex.getAliases().get(result.getName()); - if (aliasMd.indexRouting() != null) { - if (aliasMd.indexRouting().indexOf(',') != -1) { - throw new IllegalArgumentException("index/alias [" + aliasOrIndex + "] provided with routing value [" - + aliasMd.getIndexRouting() + "] that resolved to several routing values, rejecting operation"); - } - if (routing != null) { - if (routing.equals(aliasMd.indexRouting()) == false) { - throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has index routing associated with it [" - + aliasMd.indexRouting() + "], and was provided with routing value [" + routing + "], rejecting operation"); - } - } - // Alias routing overrides the parent routing (if any). - return aliasMd.indexRouting(); + AliasMetadata writeIndexAliasMetadata = writeIndex.getAliases().get(result.getName()); + if (writeIndexAliasMetadata != null) { + return resolveRouting(routing, aliasOrIndex, writeIndexAliasMetadata); + } else { + return routing; } - return routing; } /** @@ -755,6 +759,12 @@ public Map dataStreams() { .orElse(Collections.emptyMap()); } + public Map dataStreamAliases() { + return Optional.ofNullable((DataStreamMetadata) this.custom(DataStreamMetadata.TYPE)) + .map(DataStreamMetadata::getDataStreamAliases) + .orElse(Collections.emptyMap()); + } + public ImmutableOpenMap customs() { return this.customs; } @@ -1251,11 +1261,18 @@ public Builder removeIndexTemplate(String name) { } public DataStream dataStream(String dataStreamName) { - return ((DataStreamMetadata) customs.get(DataStreamMetadata.TYPE)).dataStreams().get(dataStreamName); + DataStreamMetadata dataStreamMetadata = (DataStreamMetadata) customs.get(DataStreamMetadata.TYPE); + if (dataStreamMetadata != null) { + return dataStreamMetadata.dataStreams().get(dataStreamName); + } else { + return null; + } } public Builder dataStreams(Map dataStreams) { - this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(dataStreams)); + // TODO: take into account aliases... + // (this is only used from snapshot / restore) + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(dataStreams, new HashMap<>())); return this; } @@ -1266,20 +1283,111 @@ public Builder put(DataStream dataStream) { .map(dsmd -> new HashMap<>(dsmd.dataStreams())) .orElse(new HashMap<>()); existingDataStreams.put(dataStream.getName(), dataStream); - this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams)); + Map existingDataStreamAliases = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) + .orElse(new HashMap<>()); + + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams, existingDataStreamAliases)); return this; } + public boolean put(String name, String dataStream) { + Map existingDataStream = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.dataStreams())) + .orElse(new HashMap<>()); + Map dataStreamAliases = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) + .orElse(new HashMap<>()); + + if (existingDataStream.containsKey(dataStream) == false) { + throw new IllegalArgumentException("alias [" + name + "] refers to a non existing data stream [" + dataStream + "]"); + } + + DataStreamAlias alias = dataStreamAliases.get(name); + if (alias == null) { + alias = new DataStreamAlias(name, List.of(dataStream)); + } else { + Set dataStreams = new HashSet<>(alias.getDataStreams()); + boolean added = dataStreams.add(dataStream); + if (added == false) { + return false; + } + alias = new DataStreamAlias(name, List.copyOf(dataStreams)); + } + dataStreamAliases.put(name, alias); + + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStream, dataStreamAliases)); + return true; + } + public Builder removeDataStream(String name) { Map existingDataStreams = Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) .map(dsmd -> new HashMap<>(dsmd.dataStreams())) .orElse(new HashMap<>()); existingDataStreams.remove(name); - this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams)); + + Map existingDataStreamAliases = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) + .orElse(new HashMap<>()); + + Set aliasesToDelete = new HashSet<>(); + List aliasesToUpdate = new ArrayList<>(); + for (var alias : existingDataStreamAliases.values()) { + Set dataStreams = new HashSet<>(alias.getDataStreams()); + if (dataStreams.contains(name)) { + dataStreams.remove(name); + if (dataStreams.isEmpty()) { + aliasesToDelete.add(alias.getName()); + } else { + aliasesToUpdate.add(new DataStreamAlias(alias.getName(), List.copyOf(dataStreams))); + } + } + } + for (DataStreamAlias alias : aliasesToUpdate) { + existingDataStreamAliases.put(alias.getName(), alias); + } + for (String aliasToDelete : aliasesToDelete) { + existingDataStreamAliases.remove(aliasToDelete); + } + + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams, existingDataStreamAliases)); return this; } + public boolean removeDataStreamAlias(String aliasName, String dataStreamName, boolean mustExist) { + Map dataStreamAliases = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases())) + .orElse(new HashMap<>()); + + DataStreamAlias existing = dataStreamAliases.get(aliasName); + if (mustExist && existing == null) { + throw new ResourceNotFoundException("alias [" + aliasName + "] doesn't exist"); + } else if (existing == null) { + return false; + } + Set dataStreams = new HashSet<>(existing.getDataStreams()); + dataStreams.remove(dataStreamName); + if (dataStreams.isEmpty()) { + dataStreamAliases.remove(aliasName); + } else { + dataStreamAliases.put(aliasName, + new DataStreamAlias(existing.getName(), List.copyOf(dataStreams))); + } + + Map existingDataStream = + Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) + .map(dsmd -> new HashMap<>(dsmd.dataStreams())) + .orElse(new HashMap<>()); + this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStream, dataStreamAliases)); + return true; + } + public Custom getCustom(String type) { return customs.get(type); } @@ -1527,6 +1635,16 @@ private SortedMap buildIndicesLookup() { indexToDataStreamLookup.put(i.getName(), dataStream); } } + for (DataStreamAlias alias : dataStreamMetadata.getDataStreamAliases().values()) { + List allIndicesOfAllDataStreams = alias.getDataStreams().stream() + .map(name -> dataStreamMetadata.dataStreams().get(name)) + .flatMap(ds -> ds.getIndices().stream()) + .map(index -> indices.get(index.getName())) + .collect(Collectors.toList()); + IndexAbstraction existing = indicesLookup.put(alias.getName(), + new IndexAbstraction.DataStreamAlias(alias, allIndicesOfAllDataStreams)); + assert existing == null : "duplicate data stream alias for " + alias.getName(); + } } Map> aliasToIndices = new HashMap<>(); @@ -1575,6 +1693,7 @@ static void validateDataStreams(SortedMap indicesLooku // Sanity check, because elsewhere a more user friendly error should have occurred: List conflictingAliases = indicesLookup.values().stream() .filter(ia -> ia.getType() == IndexAbstraction.Type.ALIAS) + .filter(ia -> ia instanceof IndexAbstraction.Alias) // TODO: a nicer to only select index aliases? .filter(ia -> { for (IndexMetadata index : ia.getIndices()) { if (indicesLookup.get(index.getIndex().getName()).getParentDataStream() != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index d1d7b2049b0b6..8ff7a4a9a7aef 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -110,16 +110,40 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable lookup = name -> { + IndexMetadata imd = metadata.get(name); + if (imd != null) { + return imd.getIndex().getName(); + } + DataStream dataStream = metadata.dataStream(name); + if (dataStream != null) { + return dataStream.getName(); + } + return null; + }; + + // Handle the actions that do data streams aliases separately: + DataStream dataStream = metadata.dataStream(action.getIndex()); + if (dataStream != null) { + NewAliasValidator newAliasValidator = (alias, indexRouting, filter, writeIndex) -> { + aliasValidator.validateAlias(alias, action.getIndex(), indexRouting, lookup); + }; + if (action.apply(newAliasValidator, metadata, null)) { + changed = true; + } + continue; + } + IndexMetadata index = metadata.get(action.getIndex()); if (index == null) { throw new IndexNotFoundException(action.getIndex()); } validateAliasTargetIsNotDSBackingIndex(currentState, action); NewAliasValidator newAliasValidator = (alias, indexRouting, filter, writeIndex) -> { - /* It is important that we look up the index using the metadata builder we are modifying so we can remove an - * index and replace it with an alias. */ - Function indexLookup = name -> metadata.get(name); - aliasValidator.validateAlias(alias, action.getIndex(), indexRouting, indexLookup); + aliasValidator.validateAlias(alias, action.getIndex(), indexRouting, lookup); if (Strings.hasLength(filter)) { IndexService indexService = indices.get(index.getIndex().getName()); if (indexService == null) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java index 09a39dd5b6b2d..3a0edc6199415 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -31,6 +32,7 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -65,7 +67,7 @@ public String getName() { static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String[] requestedAliases, ImmutableOpenMap> responseAliasMap, - XContentBuilder builder) throws Exception { + Map> dataStreamAliases, XContentBuilder builder) throws Exception { final Set indicesToDisplay = new HashSet<>(); final Set returnedAliasNames = new HashSet<>(); for (final ObjectObjectCursor> cursor : responseAliasMap) { @@ -132,7 +134,7 @@ static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String builder.field("status", status.getStatus()); } - for (final ObjectObjectCursor> entry : responseAliasMap) { + for (final var entry : responseAliasMap) { if (aliasesExplicitlyRequested == false || (aliasesExplicitlyRequested && indicesToDisplay.contains(entry.key))) { builder.startObject(entry.key); { @@ -147,6 +149,20 @@ static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String builder.endObject(); } } + for (var entry : dataStreamAliases.entrySet()) { + builder.startObject(entry.getKey()); + { + builder.startObject("aliases"); + { + for (DataStreamAlias alias : entry.getValue()) { + builder.startObject(alias.getName()); + builder.endObject(); + } + } + builder.endObject(); + } + builder.endObject(); + } } builder.endObject(); return new BytesRestResponse(status, builder); @@ -171,7 +187,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(GetAliasesResponse response, XContentBuilder builder) throws Exception { - return buildRestResponse(namesProvided, aliases, response.getAliases(), builder); + return buildRestResponse(namesProvided, aliases, response.getAliases(), response.getDataStreamAliases(), builder); } }); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java index ba2fd776e215b..ce801ffc57f94 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.alias.RandomAliasActionsGenerator; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -26,7 +27,6 @@ import java.util.Objects; import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomAliasAction; -import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomMap; import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomRouting; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.arrayContaining; @@ -111,7 +111,7 @@ public void testMustExistOption() { public void testParseAdd() throws IOException { String[] indices = generateRandomStringArray(10, 5, false, false); String[] aliases = generateRandomStringArray(10, 5, false, false); - Map filter = randomBoolean() ? randomMap(5) : null; + Map filter = randomBoolean() ? RandomAliasActionsGenerator.randomMap(5) : null; Object searchRouting = randomBoolean() ? randomRouting() : null; Object indexRouting = randomBoolean() ? randomBoolean() ? searchRouting : randomRouting() : null; boolean writeIndex = randomBoolean(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java index 8973bf236f705..cd6d7c94427e7 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java @@ -8,9 +8,11 @@ package org.elasticsearch.action.admin.indices.alias.get; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.AliasMetadata.Builder; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -34,7 +36,8 @@ protected Writeable.Reader instanceReader() { @Override protected GetAliasesResponse mutateInstance(GetAliasesResponse response) { - return new GetAliasesResponse(mutateAliases(response.getAliases())); + return new GetAliasesResponse(mutateAliases(response.getAliases()), + randomMap(5, 5, () -> new Tuple<>(randomAlphaOfLength(4), randomList(5, DataStreamTestHelper::randomAliasInstance)))); } private static ImmutableOpenMap> mutateAliases(ImmutableOpenMap> aliases) { @@ -75,7 +78,8 @@ private static ImmutableOpenMap> mutateAliases(Immut } private static GetAliasesResponse createTestItem() { - return new GetAliasesResponse(createIndicesAliasesMap(0, 5).build()); + return new GetAliasesResponse(mutateAliases(createIndicesAliasesMap(0, 5).build() + ), randomMap(5, 5, () -> new Tuple<>(randomAlphaOfLength(4), randomList(5, DataStreamTestHelper::randomAliasInstance)))); } private static ImmutableOpenMap.Builder> createIndicesAliasesMap(int min, int max) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index f8565133bb571..229de26d68713 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -29,12 +29,19 @@ public class TransportGetAliasesActionTests extends ESTestCase { public void testPostProcess() { + Metadata.Builder metadata = Metadata.builder(); + metadata.put(IndexMetadata.builder("a").settings(ESTestCase.settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)); + metadata.put(IndexMetadata.builder("b").settings(ESTestCase.settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)); + metadata.put(IndexMetadata.builder("c").settings(ESTestCase.settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(metadata).build(); + GetAliasesRequest request = new GetAliasesRequest(); ImmutableOpenMap> aliases = ImmutableOpenMap.>builder() .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build())) .build(); ImmutableOpenMap> result = - TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, + TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, clusterState, SystemIndexAccessLevel.NONE, null, EmptySystemIndices.INSTANCE); assertThat(result.size(), equalTo(3)); assertThat(result.get("a").size(), equalTo(0)); @@ -46,7 +53,7 @@ public void testPostProcess() { aliases = ImmutableOpenMap.>builder() .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build())) .build(); - result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, + result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, clusterState, SystemIndexAccessLevel.NONE, null, EmptySystemIndices.INSTANCE); assertThat(result.size(), equalTo(3)); assertThat(result.get("a").size(), equalTo(0)); @@ -57,7 +64,7 @@ public void testPostProcess() { aliases = ImmutableOpenMap.>builder() .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build())) .build(); - result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, + result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, clusterState, SystemIndexAccessLevel.NONE, null, EmptySystemIndices.INSTANCE); assertThat(result.size(), equalTo(1)); assertThat(result.get("b").size(), equalTo(1)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index 6902db27b2152..5a598aee80bbd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -139,7 +139,7 @@ private static ComposableIndexTemplate.DataStreamTemplate randomDataStreamTempla if (randomBoolean()) { return null; } else { - return new ComposableIndexTemplate.DataStreamTemplate(); + return DataStreamTemplateTests.randomInstance(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java index 7e081e2a6f528..7f43a85b18f16 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.test.AbstractNamedWriteableTestCase; import java.io.IOException; @@ -16,18 +17,34 @@ import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + public class DataStreamMetadataTests extends AbstractNamedWriteableTestCase { + public void testFromXContent() throws IOException { + xContentTester(this::createParser, this::createTestInstance, ToXContent.EMPTY_PARAMS, DataStreamMetadata::fromXContent) + .assertEqualsConsumer(this::assertEqualInstances) + .test(); + } + @Override protected DataStreamMetadata createTestInstance() { if (randomBoolean()) { - return new DataStreamMetadata(Collections.emptyMap()); + return new DataStreamMetadata(Map.of(), Map.of()); } Map dataStreams = new HashMap<>(); for (int i = 0; i < randomIntBetween(1, 5); i++) { dataStreams.put(randomAlphaOfLength(5), DataStreamTestHelper.randomInstance()); } - return new DataStreamMetadata(dataStreams); + + Map dataStreamsAliases = new HashMap<>(); + if (randomBoolean()) { + for (int i = 0; i < randomIntBetween(1, 5); i++) { + DataStreamAlias alias = DataStreamTestHelper.randomAliasInstance(); + dataStreamsAliases.put(alias.getName(), alias); + } + } + return new DataStreamMetadata(dataStreams, dataStreamsAliases); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java new file mode 100644 index 0000000000000..65e6efcdca6e4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.cluster.metadata; + +import java.io.IOException; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +public class DataStreamTemplateTests extends AbstractSerializingTestCase { + + @Override + protected DataStreamTemplate doParseInstance(XContentParser parser) throws IOException { + return DataStreamTemplate.PARSER.parse(parser, null); + } + + @Override + protected Writeable.Reader instanceReader() { + return DataStreamTemplate::new; + } + + @Override + protected DataStreamTemplate createTestInstance() { + return randomInstance(); + } + + public static DataStreamTemplate randomInstance() { + return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean()); + } + +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java index 1e966547feaee..2427eff63b63b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.Index; @@ -30,9 +31,12 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anySetOf; @@ -499,6 +503,30 @@ public void testAliasesForDataStreamBackingIndicesNotSupported() { "stream [foo-stream]. Data streams and their backing indices don't support alias operations.")); } + public void testDataStreamAliases() { + ClusterState state = DataStreamTestHelper.getClusterStateWithDataStreams(List.of( + new Tuple<>("logs-foobar", 1), new Tuple<>("metrics-foobar", 1)), List.of()); + + ClusterState result = service.applyAliasActions(state, List.of( + new AliasAction.AddDataStreamAlias("foobar", "logs-foobar"), + new AliasAction.AddDataStreamAlias("foobar", "metrics-foobar") + )); + assertThat(result.metadata().dataStreamAliases().get("foobar"), notNullValue()); + assertThat(result.metadata().dataStreamAliases().get("foobar").getDataStreams(), + containsInAnyOrder("logs-foobar", "metrics-foobar")); + + result = service.applyAliasActions(result, List.of( + new AliasAction.RemoveDataStreamAlias("foobar", "logs-foobar", null) + )); + assertThat(result.metadata().dataStreamAliases().get("foobar"), notNullValue()); + assertThat(result.metadata().dataStreamAliases().get("foobar").getDataStreams(), containsInAnyOrder("metrics-foobar")); + + result = service.applyAliasActions(result, List.of( + new AliasAction.RemoveDataStreamAlias("foobar", "metrics-foobar", null) + )); + assertThat(result.metadata().dataStreamAliases().get("foobar"), nullValue()); + } + private ClusterState applyHiddenAliasMix(ClusterState before, Boolean isHidden1, Boolean isHidden2) { return service.applyAliasActions(before, Arrays.asList( new AliasAction.Add("test", "alias", null, null, null, null, isHidden1), diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 959f907e54e38..2ee90b063bc25 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.cluster.ClusterModule; @@ -50,11 +51,13 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.Metadata.Builder.validateDataStreams; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; public class MetadataTests extends ESTestCase { @@ -1228,7 +1231,7 @@ public void testValidateDataStreamsAllowsPrefixedBackingIndices() { indicesLookup.put(indexMeta.getIndex().getName(), new IndexAbstraction.Index(indexMeta, dataStreamAbstraction)); } } - DataStreamMetadata dataStreamMetadata = new DataStreamMetadata(org.elasticsearch.common.collect.Map.of(dataStreamName, dataStream)); + DataStreamMetadata dataStreamMetadata = new DataStreamMetadata(org.elasticsearch.common.collect.Map.of(dataStreamName, dataStream), Map.of()); // prefixed indices with a lower generation than the data stream's generation are allowed even if the non-prefixed, matching the // data stream backing indices naming pattern, indices are already in the system @@ -1297,6 +1300,118 @@ public void testSnapshotWithMissingDataStream() { assertThat(e.getMessage(), containsString("unable to find data stream [" + missingDataStream + "]")); } + public void testDataStreamAliases() { + Metadata.Builder mdBuilder = Metadata.builder(); + + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-eu"), is(true)); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-us"), is(true)); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-au"), is(true)); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-au"), is(false)); + + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au")); + } + + public void testDataStreamReferToNonExistingDataStream() { + Metadata.Builder mdBuilder = Metadata.builder(); + + Exception e = expectThrows(IllegalArgumentException.class, () -> mdBuilder.put("logs-postgres", "logs-postgres-eu")); + assertThat(e.getMessage(), equalTo("alias [logs-postgres] refers to a non existing data stream [logs-postgres-eu]")); + + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu"); + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + } + + public void testDeleteDataStreamShouldUpdateAlias() { + Metadata.Builder mdBuilder = Metadata.builder(); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au"); + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au")); + + mdBuilder = Metadata.builder(metadata); + mdBuilder.removeDataStream("logs-postgres-us"); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-au")); + + mdBuilder = Metadata.builder(metadata); + mdBuilder.removeDataStream("logs-postgres-au"); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + + mdBuilder = Metadata.builder(metadata); + mdBuilder.removeDataStream("logs-postgres-eu"); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), nullValue()); + } + + public void testDeleteDataStreamAlias() { + Metadata.Builder mdBuilder = Metadata.builder(); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au"); + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au")); + + mdBuilder = Metadata.builder(metadata); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-us", true), is(true)); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-au")); + + mdBuilder = Metadata.builder(metadata); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-au", true), is(true)); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + + mdBuilder = Metadata.builder(metadata); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-eu", true), is(true)); + metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), nullValue()); + } + + public void testDeleteDataStreamAliasMustExists() { + Metadata.Builder mdBuilder = Metadata.builder(); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us"); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au"); + Metadata metadata = mdBuilder.build(); + assertThat(metadata.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(metadata.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au")); + + Metadata.Builder mdBuilder2 = Metadata.builder(metadata); + expectThrows(ResourceNotFoundException.class, () -> mdBuilder2.removeDataStreamAlias("logs-mysql", "logs-postgres-us", true)); + assertThat(mdBuilder2.removeDataStreamAlias("logs-mysql", "logs-postgres-us", false), is(false)); + } + public static Metadata randomMetadata() { return randomMetadata(1); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java index d797d98da6f28..11117278b96b2 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.NOT_FOUND; @@ -41,7 +42,7 @@ public void testBareRequest() throws Exception { final AliasMetadata fooAliasMetadata = AliasMetadata.builder("foo").build(); openMapBuilder.put("index", Arrays.asList(fooAliasMetadata, foobarAliasMetadata)); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(false, new String[0], openMapBuilder.build(), - xContentBuilder); + Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{},\"foobar\":{}}}}")); @@ -51,7 +52,7 @@ public void testSimpleAliasWildcardMatchingNothing() throws Exception { final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*" }, openMapBuilder.build(), - xContentBuilder); + Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); @@ -63,7 +64,7 @@ public void testMultipleAliasWildcardsSomeMatching() throws Exception { final AliasMetadata aliasMetadata = AliasMetadata.builder("foobar").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*", "foobar*" }, - openMapBuilder.build(), xContentBuilder); + openMapBuilder.build(), Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foobar\":{}}}}")); @@ -73,7 +74,7 @@ public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foob*", "-foo*" }, - openMapBuilder.build(), xContentBuilder); + openMapBuilder.build(), Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); @@ -85,7 +86,7 @@ public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { final AliasMetadata aliasMetadata = AliasMetadata.builder("foo").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo*", "-foob*" }, - openMapBuilder.build(), xContentBuilder); + openMapBuilder.build(), Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{}}}}")); @@ -104,7 +105,7 @@ public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws E } final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, aliasPattern, openMapBuilder.build(), - xContentBuilder); + Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(NOT_FOUND)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), @@ -115,7 +116,7 @@ public void testAliasWildcardsExcludeExplicitMissing() throws Exception { final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo", "foofoo", "-foo*" }, - openMapBuilder.build(), xContentBuilder); + openMapBuilder.build(), Map.of(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index 16deedead18f2..f7a2eaefb5245 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -27,6 +27,7 @@ import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; +import static org.elasticsearch.test.ESTestCase.generateRandomStringArray; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomBoolean; @@ -102,10 +103,18 @@ public static DataStream randomInstance() { return randomInstance(System::currentTimeMillis); } + public static DataStream randomInstance(String name) { + return randomInstance(name, System::currentTimeMillis); + } + public static DataStream randomInstance(LongSupplier timeProvider) { + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + return randomInstance(dataStreamName, timeProvider); + } + + public static DataStream randomInstance(String dataStreamName, LongSupplier timeProvider) { List indices = randomIndexInstances(); long generation = indices.size() + ESTestCase.randomLongBetween(1, 128); - String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(LuceneTestCase.random()))); Map metadata = null; if (randomBoolean()) { @@ -116,6 +125,13 @@ public static DataStream randomInstance(LongSupplier timeProvider) { randomBoolean(), randomBoolean(), false, timeProvider); } + public static DataStreamAlias randomAliasInstance() { + return new DataStreamAlias( + randomAlphaOfLength(5), + List.of(generateRandomStringArray(5, 5, false)) + ); + } + /** * Constructs {@code ClusterState} with the specified data streams and indices. * diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index dba8566d52ab5..24d4034512f26 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -46,6 +46,7 @@ import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtilsForTesting; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -123,6 +124,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -876,6 +878,15 @@ public static List randomList(int minListSize, int maxListSize, Supplier< return list; } + public static Map randomMap(int minListSize, int maxListSize, Supplier> valueConstructor) { + final int size = randomIntBetween(minListSize, maxListSize); + Map list = new HashMap<>(size); + for (int i = 0; i < size; i++) { + Tuple entry = valueConstructor.get(); + list.put(entry.v1(), entry.v2()); + } + return list; + } private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m", "micros", "nanos"}; diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index af3273ec0afc9..9f7611c42b68e 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -18,7 +18,9 @@ import java.util.Collections; import java.util.Map; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class DataStreamsRestIT extends ESRestTestCase { @@ -89,4 +91,108 @@ public void testAddingIndexTemplateWithAliasesAndDataStream() { Exception e = expectThrows(ResponseException.class, () -> client().performRequest(putComposableIndexTemplateRequest)); assertThat(e.getMessage(), containsString("template [my-template] has alias and data stream definitions")); } + + public void testDataStreamAliases() throws Exception { + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\"logs-*\"], \"data_stream\": {}}"); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + Request createDocRequest = new Request("POST", "/logs-myapp1/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + createDocRequest = new Request("POST", "/logs-myapp2/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + // Add logs-myapp1 -> logs & logs-myapp2 -> logs + Request updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity("{\"actions\":[{\"add\":{\"index\":\"logs-myapp1\",\"alias\":\"logs\"}}," + + "{\"add\":{\"index\":\"logs-myapp2\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + Request getAliasesRequest = new Request("GET", "/_aliases"); + Map getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertThat(getAliasesResponse.keySet(), containsInAnyOrder("logs-myapp1", "logs-myapp2")); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + + Request searchRequest = new Request("GET", "/logs/_search"); + Map searchResponse = entityAsMap(client().performRequest(searchRequest)); + assertEquals(2, XContentMapValues.extractValue("hits.total.value", searchResponse)); + + // Remove logs-myapp1 -> logs & logs-myapp2 -> logs + updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity("{\"actions\":[{\"remove\":{\"index\":\"logs-myapp1\",\"alias\":\"logs\"}}," + + "{\"remove\":{\"index\":\"logs-myapp2\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + getAliasesRequest = new Request("GET", "/_aliases"); + getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of(), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); + assertEquals(Map.of(), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/logs/_search"))); + + // Add logs-* -> logs + updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity("{\"actions\":[{\"add\":{\"index\":\"logs-*\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + getAliasesRequest = new Request("GET", "/_aliases"); + getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + + searchRequest = new Request("GET", "/logs/_search"); + searchResponse = entityAsMap(client().performRequest(searchRequest)); + assertEquals(2, XContentMapValues.extractValue("hits.total.value", searchResponse)); + + // Remove logs-* -> logs + updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity("{\"actions\":[{\"remove\":{\"index\":\"logs-*\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + getAliasesRequest = new Request("GET", "/_aliases"); + getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of(), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); + assertEquals(Map.of(), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/logs/_search"))); + } + + public void testDeleteDataStreamApiWithAliasFails() throws IOException { + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\"logs-*\"], \"data_stream\": {}}"); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + Request createDocRequest = new Request("POST", "/logs-emea/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + createDocRequest = new Request("POST", "/logs-nasa/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + Request updateAliasesRequest = new Request("POST", "/_aliases"); + updateAliasesRequest.setJsonEntity( + "{\"actions\":[{\"add\":{\"index\":\"logs-emea\",\"alias\":\"logs\"}}," + + "{\"add\":{\"index\":\"logs-nasa\",\"alias\":\"logs\"}}]}"); + assertOK(client().performRequest(updateAliasesRequest)); + + Request getAliasesRequest = new Request("GET", "/logs-*/_alias"); + Map getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-emea.aliases", getAliasesResponse)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-nasa.aliases", getAliasesResponse)); + + Exception e = expectThrows(ResponseException.class, () -> client().performRequest(new Request("DELETE", "/_data_stream/logs"))); + assertThat(e.getMessage(), containsString("The provided expression [logs] matches an alias, " + + "specify the corresponding concrete indices instead")); + + assertOK(client().performRequest(new Request("DELETE", "/_data_stream/logs-emea"))); + assertOK(client().performRequest(new Request("DELETE", "/_data_stream/logs-nasa"))); + + getAliasesRequest = new Request("GET", "/logs-*/_alias"); + assertThat(entityAsMap(client().performRequest(getAliasesRequest)), equalTo(Map.of())); + } } diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 88cfa86b91eb4..1c4f03e384a72 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; @@ -42,6 +44,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Nullable; @@ -566,7 +569,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(dataStreamName, client().admin().indices().prepareUpgrade(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareRecoveries(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareUpgradeStatus(dataStreamName), false); - verifyResolvability(dataStreamName, client().admin().indices().prepareGetAliases("dummy").addIndices(dataStreamName), true); + verifyResolvability(dataStreamName, client().admin().indices().prepareGetAliases("dummy").addIndices(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareGetFieldMappings(dataStreamName), false); verifyResolvability( dataStreamName, @@ -685,7 +688,7 @@ public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exceptio assertTrue(maybeE.isPresent()); } - public void testAliasActionsFailOnDataStreams() throws Exception { + public void testAliasActionsOnDataStreams() throws Exception { putComposableIndexTemplate("id1", List.of("metrics-foo*")); String dataStreamName = "metrics-foo"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); @@ -696,8 +699,12 @@ public void testAliasActionsFailOnDataStreams() throws Exception { .aliases("foo"); IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest(); aliasesAddRequest.addAliasAction(addAction); - Exception e = expectThrows(IndexNotFoundException.class, () -> client().admin().indices().aliases(aliasesAddRequest).actionGet()); - assertThat(e.getMessage(), equalTo("no such index [" + dataStreamName + "]")); + assertAcked(client().admin().indices().aliases(aliasesAddRequest).actionGet()); + GetAliasesResponse response = client().admin().indices().getAliases(new GetAliasesRequest()).actionGet(); + assertThat( + response.getDataStreamAliases(), + equalTo(Map.of("metrics-foo", List.of(new DataStreamAlias("foo", List.of("metrics-foo"))))) + ); } public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 61f17d8063300..b624c5cdcbf09 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -1648,18 +1648,18 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithWildcard() { final User user = new User("data-stream-tester2", "data_stream_test2"); GetAliasesRequest request = new GetAliasesRequest("*"); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams and their backing indices should _not_ be in the authorized list since the backing indices // do not match the requested pattern List dataStreams = org.elasticsearch.common.collect.List.of("logs-foo", "logs-foobar"); final List authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request); for (String dsName : dataStreams) { - assertThat(authorizedIndices, not(hasItem(dsName))); + assertThat(authorizedIndices, hasItem(dsName)); DataStream dataStream = metadata.dataStreams().get(dsName); - assertThat(authorizedIndices, not(hasItem(dsName))); + assertThat(authorizedIndices, hasItem(dsName)); for (Index i : dataStream.getIndices()) { - assertThat(authorizedIndices, not(hasItem(i.getName()))); + assertThat(authorizedIndices, hasItem(i.getName())); } } @@ -1667,11 +1667,11 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithWildcard() { // pattern ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices); for (String dsName : dataStreams) { - assertThat(resolvedIndices.getLocal(), not(hasItem(dsName))); + assertThat(resolvedIndices.getLocal(), hasItem(dsName)); DataStream dataStream = metadata.dataStreams().get(dsName); - assertThat(resolvedIndices.getLocal(), not(hasItem(dsName))); + assertThat(resolvedIndices.getLocal(), hasItem(dsName)); for (Index i : dataStream.getIndices()) { - assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName()))); + assertThat(resolvedIndices.getLocal(), hasItem(i.getName())); } } } @@ -1681,24 +1681,24 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithoutWildcard( String dataStreamName = "logs-foobar"; GetAliasesRequest request = new GetAliasesRequest(dataStreamName); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams and their backing indices should _not_ be in the authorized list since the backing indices // do not match the requested name final List authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request); - assertThat(authorizedIndices, not(hasItem(dataStreamName))); + assertThat(authorizedIndices, hasItem(dataStreamName)); DataStream dataStream = metadata.dataStreams().get(dataStreamName); - assertThat(authorizedIndices, not(hasItem(dataStreamName))); + assertThat(authorizedIndices, hasItem(dataStreamName)); for (Index i : dataStream.getIndices()) { - assertThat(authorizedIndices, not(hasItem(i.getName()))); + assertThat(authorizedIndices, hasItem(i.getName())); } // neither data streams nor their backing indices will be in the resolved list since the backing indices do not match the // requested name(s) ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices); - assertThat(resolvedIndices.getLocal(), not(hasItem(dataStreamName))); + assertThat(resolvedIndices.getLocal(), hasItem(dataStreamName)); for (Index i : dataStream.getIndices()) { - assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName()))); + assertThat(resolvedIndices.getLocal(), hasItem(i.getName())); } } @@ -1795,24 +1795,24 @@ public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildca String dataStreamName = "logs-foobar"; GetAliasesRequest request = new GetAliasesRequest(dataStreamName); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams and their backing indices should _not_ be in the authorized list since the backing indices // did not match the requested pattern and the request does not support data streams final List authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request); - assertThat(authorizedIndices, not(hasItem(dataStreamName))); + assertThat(authorizedIndices, hasItem(dataStreamName)); DataStream dataStream = metadata.dataStreams().get(dataStreamName); - assertThat(authorizedIndices, not(hasItem(dataStreamName))); + assertThat(authorizedIndices, hasItem(dataStreamName)); for (Index i : dataStream.getIndices()) { - assertThat(authorizedIndices, not(hasItem(i.getName()))); + assertThat(authorizedIndices, hasItem(i.getName())); } // neither data streams nor their backing indices will be in the resolved list since the request does not support data streams // and the backing indices do not match the requested name ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices); - assertThat(resolvedIndices.getLocal(), not(hasItem(dataStreamName))); + assertThat(resolvedIndices.getLocal(), hasItem(dataStreamName)); for (Index i : dataStream.getIndices()) { - assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName()))); + assertThat(resolvedIndices.getLocal(), hasItem(i.getName())); } } @@ -1867,7 +1867,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar String indexName = ".ds-logs-foobar-*"; GetAliasesRequest request = new GetAliasesRequest(indexName); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams should _not_ be in the authorized list but their backing indices that matched both the requested pattern // and the authorized pattern should be in the list @@ -1893,7 +1893,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAnd String indexName = ".ds-logs-foobar-*"; GetAliasesRequest request = new GetAliasesRequest(indexName); assertThat(request, instanceOf(IndicesRequest.Replaceable.class)); - assertThat(request.includeDataStreams(), is(false)); + assertThat(request.includeDataStreams(), is(true)); // data streams should _not_ be in the authorized list but a single backing index that matched the requested pattern // and the authorized name should be in the list diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml new file mode 100644 index 0000000000000..0050ee20f382c --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml @@ -0,0 +1,73 @@ +--- +"Create data stream alias": + - skip: + version: " - 7.99.99" + reason: "data streams alias not yet backported to the 7.x branch" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [events-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [events-*] + template: + settings: + index.number_of_replicas: 0 + data_stream: {} + + - do: + indices.create_data_stream: + name: events-app1 + - is_true: acknowledged + + - do: + index: + index: events-app1 + refresh: true + body: + '@timestamp': '2022-12-12' + foo: bar + + - do: + indices.create_data_stream: + name: events-app2 + - is_true: acknowledged + + - do: + index: + index: events-app2 + refresh: true + body: + '@timestamp': '2022-12-12' + foo: bar + + - do: + indices.update_aliases: + body: + actions: + - add: + index: events-app1 + alias: events + - add: + index: events-app2 + alias: events + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: events-app1 } + - match: { data_streams.1.name: events-app2 } + + - do: + indices.get_alias: {} + + - match: {events-app1.aliases.events: {}} + - match: {events-app2.aliases.events: {}} + + - do: + search: + index: events + body: { query: { match_all: {} } } + - length: { hits.hits: 2 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/50_data_streams.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/50_data_streams.yml index e5afc18095be4..c832ff9818a26 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/50_data_streams.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/50_data_streams.yml @@ -192,8 +192,8 @@ teardown: - do: headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user - indices.get_alias: - name: easy* + indices.get: + index: easy* - match: {easy-index.aliases.easy-alias: {}} - is_false: easy-data-stream1 diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/51_data_stream_aliases.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/51_data_stream_aliases.yml new file mode 100644 index 0000000000000..ac506b19ccddf --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/51_data_stream_aliases.yml @@ -0,0 +1,286 @@ +--- +setup: + - skip: + features: ["headers", "allowed_warnings"] + version: " - 7.99.99" + reason: "data stream aliases not yet backported to 7.x branch" + + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_role: + name: "ingest_events_role" + body: > + { + "indices": [ + { "names": ["events*"], "privileges": ["create_doc", "create_index"] } + ] + } + + - do: + security.put_role: + name: "query_events_role" + body: > + { + "indices": [ + { "names": ["app1", "events*"], "privileges": ["read", "view_index_metadata", "monitor"] } + ] + } + + - do: + security.put_role: + name: "admin_events_role" + body: > + { + "indices": [ + { "names": ["events*"], "privileges": ["read", "write", "manage"] } + ] + } + + - do: + security.put_role: + name: "ingest_logs_role" + body: > + { + "indices": [ + { "names": ["logs*"], "privileges": ["create_doc", "create_index"] } + ] + } + + - do: + security.put_role: + name: "query_logs_role" + body: > + { + "indices": [ + { "names": ["app1", "logs*"], "privileges": ["read", "view_index_metadata", "monitor"] } + ] + } + + - do: + security.put_role: + name: "admin_logs_role" + body: > + { + "indices": [ + { "names": ["logs*"], "privileges": ["read", "write", "manage"] } + ] + } + + - do: + security.put_user: + username: "test_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "ingest_events_role", "query_events_role" ], + "full_name" : "user with privileges on event data streams" + } + + - do: + security.put_user: + username: "no_authz_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "ingest_logs_role", "query_logs_role" ], + "full_name" : "user with privileges on logs data streams" + } + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [events-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [events-*] + template: + mappings: + properties: + '@timestamp': + type: date + 'foo': + type: keyword + data_stream: {} + + - do: + allowed_warnings: + - "index template [my-template2] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [logs-*] + template: + mappings: + properties: + '@timestamp': + type: date + 'foo': + type: keyword + data_stream: {} + +--- +teardown: + - do: + security.delete_user: + username: "test_user" + ignore: 404 + + - do: + security.delete_user: + username: "test_user2" + ignore: 404 + + - do: + security.delete_role: + name: "ingest_events_role" + ignore: 404 + + - do: + security.delete_role: + name: "query_events_role" + ignore: 404 + + - do: + security.delete_role: + name: "admin_events_role" + ignore: 404 + + - do: + security.delete_role: + name: "ingest_logs_role" + ignore: 404 + + - do: + security.delete_role: + name: "query_logs_role" + ignore: 404 + + - do: + security.delete_role: + name: "admin_logs_role" + ignore: 404 + +--- +"Basic read authorization test": + - skip: + features: ["headers", "allowed_warnings"] + version: " - 7.99.99" + reason: "data stream aliases not yet backported to 7.x branch" + + - do: + index: + index: events-app1 + refresh: true + body: + '@timestamp': '2022-12-12' + foo: bar + + - do: + index: + index: logs-app1 + refresh: true + body: + '@timestamp': '2022-12-12' + bar: baz + + - do: + indices.update_aliases: + body: + actions: + - add: + index: events-app1 + alias: app1 + - add: + index: logs-app1 + alias: app1 + + # app1 allows access to both data streams for this user irregardless whether privileges have been granted on data streams + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: app1 + - match: { hits.total: 2 } + - match: { hits.hits.0._source.foo: bar } + - match: { hits.hits.1._source.bar: baz } + + # this user is authorized to use events-app1 data stream directly + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: events-app1 + - match: { hits.total: 1 } + - match: { hits.hits.0._source.foo: bar } + + # this user is authorized to use a backing index of events-app1 data stream directly + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: .ds-events-app1* + - match: { hits.total: 1 } + - match: { hits.hits.0._source.foo: bar } + + # this user isn't authorized to use logs-app1 data stream directly + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + catch: forbidden + search: + rest_total_hits_as_int: true + index: logs-app1 + + # this user isn't authorized to use a backing index of logs-app1 data stream directly + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: .ds-logs-app1* + - match: { hits.total: 0 } + + # app1 allows access to both data streams for this user irregardless whether privileges have been granted on data streams + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + search: + rest_total_hits_as_int: true + index: app1 + - match: { hits.total: 2 } + - match: { hits.hits.0._source.foo: bar } + - match: { hits.hits.1._source.bar: baz } + + # this user isn't authorized to use events-app1 data stream directly + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + catch: forbidden + search: + rest_total_hits_as_int: true + index: events-app1 + + # this user isn't authorized to use a backing index of events-app1 data stream directly + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + search: + rest_total_hits_as_int: true + index: .ds-events-app1* + - match: { hits.total: 0 } + + # this user is authorized to use logs-app1 data stream directly + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + search: + rest_total_hits_as_int: true + index: logs-app1 + - match: { hits.total: 1 } + - match: { hits.hits.0._source.bar: baz } + + # this user is authorized to use a backing index of logs-app1 data stream directly + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + search: + rest_total_hits_as_int: true + index: .ds-logs-app1* + - match: { hits.total: 1 } + - match: { hits.hits.0._source.bar: baz }