From 6815f50a80b6ee0db94f267a5b1412da52ffd53f Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 6 Apr 2021 08:42:27 -0600 Subject: [PATCH 01/22] Add Fleet action results system data stream This commit adds support for system data streams and also the first use of a system data stream with the fleet action results data stream. --- .../client/indices/DataStream.java | 19 +- .../alias/get/TransportGetAliasesAction.java | 22 +- .../indices/create/AutoCreateAction.java | 33 ++- .../CreateIndexClusterStateUpdateRequest.java | 12 + .../action/support/AutoCreateIndex.java | 2 +- .../cluster/metadata/DataStream.java | 42 +++- .../metadata/IndexNameExpressionResolver.java | 35 +-- .../MetadataCreateDataStreamService.java | 78 ++++-- .../metadata/MetadataCreateIndexService.java | 84 ++++++- .../MetadataIndexTemplateService.java | 34 ++- .../MetadataMigrateToDataStreamService.java | 21 +- .../SystemIndexMetadataUpgradeService.java | 8 +- .../indices/SystemDataStreamDescriptor.java | 90 +++++++ .../elasticsearch/indices/SystemIndices.java | 226 ++++++++++++++++-- .../java/org/elasticsearch/node/Node.java | 41 ++-- .../plugins/SystemIndexPlugin.java | 5 + .../elasticsearch/rest/RestController.java | 4 +- .../snapshots/RestoreService.java | 2 +- .../get/TransportGetAliasesActionTests.java | 8 +- .../DateMathExpressionResolverTests.java | 5 +- .../IndexNameExpressionResolverTests.java | 6 +- .../MetadataCreateDataStreamServiceTests.java | 20 +- ...tadataMigrateToDataStreamServiceTests.java | 24 +- .../WildcardExpressionResolverTests.java | 2 +- .../metadata/DataStreamTestHelper.java | 5 +- .../core/ilm/GenerateSnapshotNameStep.java | 2 +- .../main/resources/fleet-actions-results.json | 54 +++++ .../CreateDataStreamTransportAction.java | 10 +- .../MigrateToDataStreamTransportAction.java | 8 +- .../xpack/fleet/FleetSystemIndicesIT.java | 7 + .../org/elasticsearch/xpack/fleet/Fleet.java | 31 +++ 31 files changed, 762 insertions(+), 178 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java create mode 100644 x-pack/plugin/core/src/main/resources/fleet-actions-results.json diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java index 32f62aeeafa0b..691949e5eeb2d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java @@ -26,6 +26,7 @@ public final class DataStream { private final List indices; private final long generation; private final boolean hidden; + private final boolean system; ClusterHealthStatus dataStreamStatus; @Nullable String indexTemplate; @@ -36,7 +37,7 @@ public final class DataStream { public DataStream(String name, String timeStampField, List indices, long generation, ClusterHealthStatus dataStreamStatus, @Nullable String indexTemplate, @Nullable String ilmPolicyName, @Nullable Map metadata, - boolean hidden) { + boolean hidden, boolean system) { this.name = name; this.timeStampField = timeStampField; this.indices = indices; @@ -46,6 +47,7 @@ public DataStream(String name, String timeStampField, List indices, long this.ilmPolicyName = ilmPolicyName; this.metadata = metadata; this.hidden = hidden; + this.system = system; } public String getName() { @@ -84,6 +86,10 @@ public boolean isHidden() { return hidden; } + public boolean isSystem() { + return system; + } + public static final ParseField NAME_FIELD = new ParseField("name"); public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field"); public static final ParseField INDICES_FIELD = new ParseField("indices"); @@ -93,6 +99,7 @@ public boolean isHidden() { public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy"); public static final ParseField METADATA_FIELD = new ParseField("_meta"); public static final ParseField HIDDEN_FIELD = new ParseField("hidden"); + public static final ParseField SYSTEM_FIELD = new ParseField("system"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", @@ -109,7 +116,9 @@ public boolean isHidden() { Map metadata = (Map) args[7]; Boolean hidden = (Boolean) args[8]; hidden = hidden != null && hidden; - return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata, hidden); + boolean system = args[9] != null && (boolean) args[9]; + return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata, hidden, + system); }); static { @@ -122,6 +131,7 @@ public boolean isHidden() { PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ILM_POLICY_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), SYSTEM_FIELD); } public static DataStream fromXContent(XContentParser parser) throws IOException { @@ -138,6 +148,8 @@ public boolean equals(Object o) { timeStampField.equals(that.timeStampField) && indices.equals(that.indices) && dataStreamStatus == that.dataStreamStatus && + hidden == that.hidden && + system == that.system && Objects.equals(indexTemplate, that.indexTemplate) && Objects.equals(ilmPolicyName, that.ilmPolicyName) && Objects.equals(metadata, that.metadata); @@ -145,6 +157,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, timeStampField, indices, generation, dataStreamStatus, indexTemplate, ilmPolicyName, metadata); + return Objects.hash(name, timeStampField, indices, generation, dataStreamStatus, indexTemplate, ilmPolicyName, metadata, hidden, + system); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index dc4203392160a..d1c284274eb5c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SystemIndexAccessLevel; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.inject.Inject; @@ -24,6 +23,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -66,11 +66,9 @@ protected void masterOperation(Task task, GetAliasesRequest request, ClusterStat concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request); } final SystemIndexAccessLevel systemIndexAccessLevel = indexNameExpressionResolver.getSystemIndexAccessLevel(); - final String elasticProduct = - threadPool.getThreadContext().getHeader(IndexNameExpressionResolver.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY); ImmutableOpenMap> aliases = state.metadata().findAliases(request, concreteIndices); listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases, state, - systemIndexAccessLevel, elasticProduct, systemIndices))); + systemIndexAccessLevel, threadPool.getThreadContext(), systemIndices))); } /** @@ -79,7 +77,7 @@ protected void masterOperation(Task task, GetAliasesRequest request, ClusterStat static ImmutableOpenMap> postProcess(GetAliasesRequest request, String[] concreteIndices, ImmutableOpenMap> aliases, ClusterState state, SystemIndexAccessLevel systemIndexAccessLevel, - String elasticProduct, SystemIndices systemIndices) { + ThreadContext threadContext, SystemIndices systemIndices) { boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0; ImmutableOpenMap.Builder> mapBuilder = ImmutableOpenMap.builder(aliases); for (String index : concreteIndices) { @@ -90,19 +88,19 @@ static ImmutableOpenMap> postProcess(GetAliasesReque } final ImmutableOpenMap> finalResponse = mapBuilder.build(); if (systemIndexAccessLevel != SystemIndexAccessLevel.ALL) { - checkSystemIndexAccess(request, systemIndices, state, finalResponse, systemIndexAccessLevel, elasticProduct); + checkSystemIndexAccess(request, systemIndices, state, finalResponse, systemIndexAccessLevel, threadContext); } return finalResponse; } private static void checkSystemIndexAccess(GetAliasesRequest request, SystemIndices systemIndices, ClusterState state, ImmutableOpenMap> aliasesMap, - SystemIndexAccessLevel systemIndexAccessLevel, String elasticProduct) { + SystemIndexAccessLevel systemIndexAccessLevel, ThreadContext threadContext) { final Predicate systemIndexAccessAllowPredicate; if (systemIndexAccessLevel == SystemIndexAccessLevel.NONE) { systemIndexAccessAllowPredicate = indexMetadata -> false; } else if (systemIndexAccessLevel == SystemIndexAccessLevel.RESTRICTED) { - systemIndexAccessAllowPredicate = systemIndices.getProductSystemIndexMetadataPredicate(elasticProduct); + systemIndexAccessAllowPredicate = systemIndices.getProductSystemIndexMetadataPredicate(threadContext); } else { throw new IllegalArgumentException("Unexpected system index access level: " + systemIndexAccessLevel); } @@ -122,23 +120,23 @@ private static void checkSystemIndexAccess(GetAliasesRequest request, SystemIndi "this request accesses system indices: {}, but in a future major version, direct access to system " + "indices will be prevented by default", systemIndicesNames); } else { - checkSystemAliasAccess(request, systemIndices, systemIndexAccessLevel, elasticProduct); + checkSystemAliasAccess(request, systemIndices, systemIndexAccessLevel, threadContext); } } private static void checkSystemAliasAccess(GetAliasesRequest request, SystemIndices systemIndices, - SystemIndexAccessLevel systemIndexAccessLevel, String elasticProduct) { + SystemIndexAccessLevel systemIndexAccessLevel, ThreadContext threadContext) { final Predicate systemIndexAccessAllowPredicate; if (systemIndexAccessLevel == SystemIndexAccessLevel.NONE) { systemIndexAccessAllowPredicate = name -> true; } else if (systemIndexAccessLevel == SystemIndexAccessLevel.RESTRICTED) { - systemIndexAccessAllowPredicate = systemIndices.getProductSystemIndexNamePredicate(elasticProduct).negate(); + systemIndexAccessAllowPredicate = systemIndices.getProductSystemIndexNamePredicate(threadContext).negate(); } else { throw new IllegalArgumentException("Unexpected system index access level: " + systemIndexAccessLevel); } final List systemAliases = Arrays.stream(request.aliases()) - .filter(systemIndices::isSystemIndex) + .filter(systemIndices::isSystemName) .filter(systemIndexAccessAllowPredicate) .collect(Collectors.toList()); if (systemAliases.isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index ead911c31df6d..f55308515813b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; @@ -111,11 +112,17 @@ protected void masterOperation(Task task, @Override public ClusterState execute(ClusterState currentState) throws Exception { + final SystemDataStreamDescriptor dataStreamDescriptor = + systemIndices.validateDataStreamAccess(request.index(), threadPool.getThreadContext()); + final boolean isSystemDataStream = dataStreamDescriptor != null; + final boolean isSystemIndex = isSystemDataStream == false && systemIndices.isSystemIndex(request.index()); final ComposableIndexTemplate template = resolveTemplate(request, currentState.metadata()); + final boolean isDataStream = isSystemIndex == false && + (isSystemDataStream || (template != null && template.getDataStreamTemplate() != null)); - if (template != null && template.getDataStreamTemplate() != null) { + if (isDataStream) { // This expression only evaluates to true when the argument is non-null and false - if (Boolean.FALSE.equals(template.getAllowAutoCreate())) { + if (isSystemDataStream == false && Boolean.FALSE.equals(template.getAllowAutoCreate())) { throw new IndexNotFoundException( "composable template " + template.indexPatterns() + " forbids index auto creation" ); @@ -123,6 +130,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { CreateDataStreamClusterStateUpdateRequest createRequest = new CreateDataStreamClusterStateUpdateRequest( request.index(), + dataStreamDescriptor, request.masterNodeTimeout(), request.timeout() ); @@ -132,21 +140,26 @@ public ClusterState execute(ClusterState currentState) throws Exception { } else { String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); indexNameRef.set(indexName); + if (isSystemIndex) { + if (indexName.equals(request.index()) == false) { + throw new IllegalStateException("system indices do not support date math expressions"); + } + } else { + // This will throw an exception if the index does not exist and creating it is prohibited + final boolean shouldAutoCreate = autoCreateIndex.shouldAutoCreate(indexName, currentState); - // This will throw an exception if the index does not exist and creating it is prohibited - final boolean shouldAutoCreate = autoCreateIndex.shouldAutoCreate(indexName, currentState); - - if (shouldAutoCreate == false) { - // The index already exists. - return currentState; + if (shouldAutoCreate == false) { + // The index already exists. + return currentState; + } } final SystemIndexDescriptor mainDescriptor = systemIndices.findMatchingDescriptor(indexName); - final boolean isSystemIndex = mainDescriptor != null && mainDescriptor.isAutomaticallyManaged(); + final boolean isManagedSystemIndex = mainDescriptor != null && mainDescriptor.isAutomaticallyManaged(); final CreateIndexClusterStateUpdateRequest updateRequest; - if (isSystemIndex) { + if (isManagedSystemIndex) { final SystemIndexDescriptor descriptor = mainDescriptor.getDescriptorCompatibleWith(state.nodes().getSmallestNonClientNodeVersion()); if (descriptor == null) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index 898c7b0b83b7b..f294929706fa2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import java.util.HashSet; import java.util.Set; @@ -33,6 +34,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private Index recoverFrom; private ResizeType resizeType; private boolean copySettings; + private SystemDataStreamDescriptor systemDataStreamDescriptor; private Settings settings = Settings.Builder.EMPTY_SETTINGS; @@ -93,6 +95,11 @@ public CreateIndexClusterStateUpdateRequest nameResolvedInstant(long nameResolve return this; } + public CreateIndexClusterStateUpdateRequest systemDataStreamDescriptor(SystemDataStreamDescriptor systemDataStreamDescriptor) { + this.systemDataStreamDescriptor = systemDataStreamDescriptor; + return this; + } + public String cause() { return cause; } @@ -121,6 +128,10 @@ public Index recoverFrom() { return recoverFrom; } + public SystemDataStreamDescriptor systemDataStreamDescriptor() { + return systemDataStreamDescriptor; + } + /** * The name that was provided by the user. This might contain a date math expression. * @see IndexMetadata#SETTING_INDEX_PROVIDED_NAME @@ -176,6 +187,7 @@ public String toString() { ", aliases=" + aliases + ", blocks=" + blocks + ", waitForActiveShards=" + waitForActiveShards + + ", systemDataStreamDescriptor=" + systemDataStreamDescriptor + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index 6dc49eb76f03e..3a79ac54d9a25 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -59,7 +59,7 @@ public boolean shouldAutoCreate(String index, ClusterState state) { } // Always auto-create system indexes - if (systemIndices.isSystemIndex(index)) { + if (systemIndices.isSystemName(index)) { return true; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 80b52d3a26ff3..fa564e657d9f9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; @@ -45,19 +46,25 @@ public final class DataStream extends AbstractDiffable implements To private final Map metadata; private final boolean hidden; private final boolean replicated; + private final boolean system; public DataStream(String name, TimestampField timeStampField, List indices, long generation, Map metadata) { - this(name, timeStampField, indices, generation, metadata, false, false); + this(name, timeStampField, indices, generation, metadata, false, false, false); } public DataStream(String name, TimestampField timeStampField, List indices, long generation, Map metadata, boolean hidden, boolean replicated) { - this(name, timeStampField, indices, generation, metadata, hidden, replicated, System::currentTimeMillis); + this(name, timeStampField, indices, generation, metadata, hidden, replicated, false, System::currentTimeMillis); + } + + public DataStream(String name, TimestampField timeStampField, List indices, long generation, Map metadata, + boolean hidden, boolean replicated, boolean system) { + this(name, timeStampField, indices, generation, metadata, hidden, replicated, system, System::currentTimeMillis); } // visible for testing DataStream(String name, TimestampField timeStampField, List indices, long generation, Map metadata, - boolean hidden, boolean replicated, LongSupplier timeProvider) { + boolean hidden, boolean replicated, boolean system, LongSupplier timeProvider) { this.name = name; this.timeStampField = timeStampField; this.indices = Collections.unmodifiableList(indices); @@ -66,6 +73,7 @@ public DataStream(String name, TimestampField timeStampField, List indice this.hidden = hidden; this.replicated = replicated; this.timeProvider = timeProvider; + this.system = system; assert indices.size() > 0; } @@ -112,6 +120,10 @@ public boolean isReplicated() { return replicated; } + public boolean isSystem() { + return system; + } + /** * Performs a rollover on a {@code DataStream} instance and returns a new instance containing * the updated list of backing indices and incremented generation. @@ -135,7 +147,7 @@ public DataStream rollover(Metadata clusterMetadata, String writeIndexUuid) { newWriteIndexName = DataStream.getDefaultBackingIndexName(getName(), ++generation, currentTimeMillis); } while (clusterMetadata.getIndicesLookup().containsKey(newWriteIndexName)); backingIndices.add(new Index(newWriteIndexName, writeIndexUuid)); - return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden, replicated); + return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden, replicated, system); } /** @@ -149,7 +161,7 @@ public DataStream removeBackingIndex(Index index) { List backingIndices = new ArrayList<>(indices); backingIndices.remove(index); assert backingIndices.size() == indices.size() - 1; - return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden, replicated); + return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden, replicated, system); } /** @@ -174,11 +186,11 @@ public DataStream replaceBackingIndex(Index existingBackingIndex, Index newBacki "it is the write index", existingBackingIndex.getName(), name)); } backingIndices.set(backingIndexPosition, newBackingIndex); - return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden, replicated); + return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden, replicated, system); } public DataStream promoteDataStream() { - return new DataStream(name, timeStampField, indices, getGeneration(), metadata, hidden, false, timeProvider); + return new DataStream(name, timeStampField, indices, getGeneration(), metadata, hidden, false, system, timeProvider); } /** @@ -208,7 +220,8 @@ public DataStream snapshot(Collection indicesInSnapshot) { generation, metadata == null ? null : new HashMap<>(metadata), hidden, - replicated + replicated, + system ); } @@ -240,7 +253,9 @@ public static String getDefaultBackingIndexName(String dataStreamName, long gene public DataStream(StreamInput in) throws IOException { this(in.readString(), new TimestampField(in), in.readList(Index::new), in.readVLong(), - in.readMap(), in.readBoolean(), in.readBoolean()); + in.readMap(), in.readBoolean(), in.readBoolean(), + in.getVersion().onOrAfter(Version.V_8_0_0) && in.readBoolean() // TODO change to V_7_13_0 + ); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -256,6 +271,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(metadata); out.writeBoolean(hidden); out.writeBoolean(replicated); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO change to V_7_13_0 + out.writeBoolean(system); + } } public static final ParseField NAME_FIELD = new ParseField("name"); @@ -265,11 +283,13 @@ public void writeTo(StreamOutput out) throws IOException { public static final ParseField METADATA_FIELD = new ParseField("_meta"); public static final ParseField HIDDEN_FIELD = new ParseField("hidden"); public static final ParseField REPLICATED_FIELD = new ParseField("replicated"); + public static final ParseField SYSTEM_FIELD = new ParseField("system"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", args -> new DataStream((String) args[0], (TimestampField) args[1], (List) args[2], (Long) args[3], - (Map) args[4], args[5] != null && (boolean) args[5], args[6] != null && (boolean) args[6])); + (Map) args[4], args[5] != null && (boolean) args[5], args[6] != null && (boolean) args[6], + args[7] != null && (boolean) args[7])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); @@ -279,6 +299,7 @@ public void writeTo(StreamOutput out) throws IOException { PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), REPLICATED_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), SYSTEM_FIELD); } public static DataStream fromXContent(XContentParser parser) throws IOException { @@ -297,6 +318,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(HIDDEN_FIELD.getPreferredName(), hidden); builder.field(REPLICATED_FIELD.getPreferredName(), replicated); + builder.field(SYSTEM_FIELD.getPreferredName(), system); builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index badb07f10ada1..9c54627f4e1fc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -31,6 +30,7 @@ import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import java.time.Instant; import java.time.ZoneId; @@ -57,8 +57,6 @@ public class IndexNameExpressionResolver { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IndexNameExpressionResolver.class); public static final String EXCLUDED_DATA_STREAMS_KEY = "es.excluded_ds"; - public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed"; - public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin"; public static final Version SYSTEM_INDEX_ENFORCEMENT_VERSION = Version.V_8_0_0; private final DateMathExpressionResolver dateMathExpressionResolver = new DateMathExpressionResolver(); @@ -323,9 +321,7 @@ private void checkSystemIndexAccess(Context context, Metadata metadata, Set true; } else { // everything other than allowed should be included in the deprecation message - systemIndexAccessLevelPredicate = systemIndices - .getProductSystemIndexMetadataPredicate(threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY)) - .negate(); + systemIndexAccessLevelPredicate = systemIndices.getProductSystemIndexMetadataPredicate(threadContext).negate(); } final List resolvedSystemIndices = concreteIndices.stream() .map(metadata::index) @@ -714,33 +710,8 @@ boolean isPatternMatchingAllIndices(Metadata metadata, String[] indicesOrAliases return false; } - /** - * Determines what level of system index access should be allowed in the current context. - * - * @return {@link SystemIndexAccessLevel#ALL} if unrestricted system index access should be allowed, - * {@link SystemIndexAccessLevel#RESTRICTED} if a subset of system index access should be allowed, or - * {@link SystemIndexAccessLevel#NONE} if no system index access should be allowed. - */ public SystemIndexAccessLevel getSystemIndexAccessLevel() { - final String headerValue = threadContext.getHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY); - final String productHeaderValue = threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY); - - final boolean allowed = Booleans.parseBoolean(headerValue, true); - if (allowed) { - if (productHeaderValue != null) { - return SystemIndexAccessLevel.RESTRICTED; - } else { - return SystemIndexAccessLevel.ALL; - } - } else { - return SystemIndexAccessLevel.NONE; - } - } - - public enum SystemIndexAccessLevel { - ALL, - NONE, - RESTRICTED + return systemIndices.getSystemIndexAccessLevel(threadContext); } public static class Context { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 05d6795339c2e..64c82cbfee5d1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -24,11 +24,13 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectPath; import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; @@ -47,6 +49,7 @@ public class MetadataCreateDataStreamService { private final ClusterService clusterService; private final ActiveShardsObserver activeShardsObserver; private final MetadataCreateIndexService metadataCreateIndexService; + private final ThreadContext threadContext; public MetadataCreateDataStreamService(ThreadPool threadPool, ClusterService clusterService, @@ -54,6 +57,7 @@ public MetadataCreateDataStreamService(ThreadPool threadPool, this.clusterService = clusterService; this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); this.metadataCreateIndexService = metadataCreateIndexService; + this.threadContext = threadPool.getThreadContext(); } public void createDataStream(CreateDataStreamClusterStateUpdateRequest request, @@ -80,7 +84,7 @@ public void createDataStream(CreateDataStreamClusterStateUpdateRequest request, new AckedClusterStateUpdateTask(Priority.HIGH, request, listener) { @Override public ClusterState execute(ClusterState currentState) throws Exception { - ClusterState clusterState = createDataStream(metadataCreateIndexService, currentState, request); + ClusterState clusterState = createDataStream(metadataCreateIndexService, currentState, request, threadContext); firstBackingIndexRef.set(clusterState.metadata().dataStreams().get(request.name).getIndices().get(0).getName()); return clusterState; } @@ -88,26 +92,53 @@ public ClusterState execute(ClusterState currentState) throws Exception { } public ClusterState createDataStream(CreateDataStreamClusterStateUpdateRequest request, ClusterState current) throws Exception { - return createDataStream(metadataCreateIndexService, current, request); + return createDataStream(metadataCreateIndexService, current, request, threadContext); } - public static final class CreateDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest { + public static final class CreateDataStreamClusterStateUpdateRequest + extends ClusterStateUpdateRequest { private final String name; + private final SystemDataStreamDescriptor descriptor; public CreateDataStreamClusterStateUpdateRequest(String name, TimeValue masterNodeTimeout, TimeValue timeout) { + this(name, null, masterNodeTimeout, timeout); + } + + public CreateDataStreamClusterStateUpdateRequest(String name, + SystemDataStreamDescriptor systemDataStreamDescriptor, + TimeValue masterNodeTimeout, + TimeValue timeout) { this.name = name; + this.descriptor = systemDataStreamDescriptor; masterNodeTimeout(masterNodeTimeout); ackTimeout(timeout); } + + public boolean isSystem() { + return descriptor != null; + } + + public SystemDataStreamDescriptor getSystemDataStreamDescriptor() { + return descriptor; + } } static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, ClusterState currentState, - CreateDataStreamClusterStateUpdateRequest request) throws Exception { - return createDataStream(metadataCreateIndexService, currentState, request.name, List.of(), null); + CreateDataStreamClusterStateUpdateRequest request, + ThreadContext threadContext) throws Exception { + return createDataStream( + metadataCreateIndexService, + currentState, + request.name, + List.of(), + null, + request.getSystemDataStreamDescriptor(), + threadContext + ); } /** @@ -124,11 +155,22 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn ClusterState currentState, String dataStreamName, List backingIndices, - IndexMetadata writeIndex) throws Exception + IndexMetadata writeIndex, + ThreadContext threadContext) throws Exception { + assert metadataCreateIndexService.getSystemIndices().isSystemDataStream(dataStreamName) == false : + "dataStream [" + dataStreamName + "] is system but no system descriptor was provided!"; + return createDataStream(metadataCreateIndexService, currentState, dataStreamName, backingIndices, writeIndex, null, threadContext); + } + + private static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, + ClusterState currentState, + String dataStreamName, + List backingIndices, + IndexMetadata writeIndex, + SystemDataStreamDescriptor systemDataStreamDescriptor, + ThreadContext threadContext) throws Exception { - if (writeIndex == null) { - Objects.requireNonNull(metadataCreateIndexService); - } + Objects.requireNonNull(metadataCreateIndexService); Objects.requireNonNull(currentState); Objects.requireNonNull(backingIndices); if (currentState.metadata().dataStreams().containsKey(dataStreamName)) { @@ -145,15 +187,23 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '" + DataStream.BACKING_INDEX_PREFIX + "'"); } - - ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, currentState.metadata()); +; + final boolean isSystem = systemDataStreamDescriptor != null; + final ComposableIndexTemplate template = isSystem ? + systemDataStreamDescriptor.getComposableIndexTemplate() : + lookupTemplateForDataStream(dataStreamName, currentState.metadata()); if (writeIndex == null) { String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); CreateIndexClusterStateUpdateRequest createIndexRequest = new CreateIndexClusterStateUpdateRequest("initialize_data_stream", firstBackingIndexName, firstBackingIndexName) .dataStreamName(dataStreamName) - .settings(Settings.builder().put("index.hidden", true).build()); + .systemDataStreamDescriptor(systemDataStreamDescriptor); + + if (isSystem == false) { + createIndexRequest.settings(Settings.builder().put("index.hidden", true).build()); + } + try { currentState = metadataCreateIndexService.applyCreateIndexRequest(currentState, createIndexRequest, false); } catch (ResourceAlreadyExistsException e) { @@ -172,9 +222,9 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName); List dsBackingIndices = backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()); dsBackingIndices.add(writeIndex.getIndex()); - boolean hidden = template.getDataStreamTemplate().isHidden(); + boolean hidden = isSystem ? false : template.getDataStreamTemplate().isHidden(); DataStream newDataStream = new DataStream(dataStreamName, timestampField, dsBackingIndices, 1L, - template.metadata() != null ? Map.copyOf(template.metadata()) : null, hidden, false); + template.metadata() != null ? Map.copyOf(template.metadata()) : null, hidden, isSystem); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); logger.info("adding data stream [{}] with write index [{}] and backing indices [{}]", dataStreamName, writeIndex.getIndex().getName(), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 35565d2b971ef..af1dc81d1c7e5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -63,7 +63,6 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.indices.ShardLimitValidator; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.threadpool.ThreadPool; @@ -78,6 +77,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; @@ -98,6 +98,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; /** * Service responsible for submitting create index requests @@ -192,11 +193,9 @@ public void validateIndexName(String index, ClusterState state) { public boolean validateDotIndex(String index, @Nullable Boolean isHidden) { boolean isSystem = false; if (index.charAt(0) == '.') { - SystemIndexDescriptor matchingDescriptor = systemIndices.findMatchingDescriptor(index); - if (matchingDescriptor != null) { - logger.trace("index [{}] is a system index because it matches index pattern [{}] with description [{}]", - index, matchingDescriptor.getIndexPattern(), matchingDescriptor.getDescription()); - isSystem = true; + isSystem = systemIndices.isSystemName(index); + if (isSystem) { + logger.trace("index [{}] is a system index", index); } else if (isHidden) { logger.trace("index [{}] is a hidden index", index); } else { @@ -209,6 +208,10 @@ public boolean validateDotIndex(String index, @Nullable Boolean isHidden) { return isSystem; } + public SystemIndices getSystemIndices() { + return systemIndices; + } + /** * Validate the name for an index or alias against some static rules. */ @@ -337,6 +340,10 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd // The backing index may have a different name or prefix than the data stream name. final String name = request.dataStreamName() != null ? request.dataStreamName() : request.index(); + + if (request.systemDataStreamDescriptor() != null) { + return applyCreateIndexRequestForSystemDataStream(currentState, request, silent, metadataTransformer); + } // Check to see if a v2 template matched final String v2Template = MetadataIndexTemplateService.findV2Template(currentState.metadata(), name, isHiddenFromRequest == null ? false : isHiddenFromRequest); @@ -466,7 +473,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c templates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), xContentRegistry)); final Settings aggregatedIndexSettings = - aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(templates), + aggregateIndexSettings(currentState, request, resolveSettings(templates), null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -500,7 +507,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu collectV2Mappings(request.mappings(), currentState, templateName, xContentRegistry, request.index()); final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, - MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName), + resolveSettings(currentState.metadata(), templateName), null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -515,15 +522,71 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu Collections.singletonList(templateName), metadataTransformer); } + private ClusterState applyCreateIndexRequestForSystemDataStream(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final BiConsumer metadataTransformer) + throws Exception { + Objects.requireNonNull(request.systemDataStreamDescriptor()); + logger.debug("applying create index request for system data stream [{}]", request.systemDataStreamDescriptor()); + + ComposableIndexTemplate template = request.systemDataStreamDescriptor().getComposableIndexTemplate(); + if (request.dataStreamName() == null && template.getDataStreamTemplate() != null) { + throw new IllegalArgumentException("cannot create index with name [" + request.index() + + "], because it matches with a system data stream"); + } + + final Map componentTemplates = request.systemDataStreamDescriptor().getComponentTemplates(); + final List> mappings = + collectSystemV2Mappings(template, componentTemplates, xContentRegistry, request.index()); + + final Settings aggregatedIndexSettings = aggregateIndexSettings( + currentState, + request, + resolveSettings(template, componentTemplates), + null, + settings, + indexScopedSettings, + shardLimitValidator, + indexSettingProviders + ); + final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); + final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); + + return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, + indexService -> resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(template, componentTemplates, null), currentState.metadata(), + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + aliasValidator, xContentRegistry, indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), + indexService.dateMathExpressionResolverAt(request.getNameResolvedAt())), + List.of(), metadataTransformer); + } + + private static List> collectSystemV2Mappings(final ComposableIndexTemplate composableIndexTemplate, + final Map componentTemplates, + final NamedXContentRegistry xContentRegistry, + final String indexName) throws Exception { + List templateMappings = + MetadataIndexTemplateService.collectMappings(composableIndexTemplate, componentTemplates, indexName, xContentRegistry); + return collectV2Mappings("{}", templateMappings, xContentRegistry); + } + public static List> collectV2Mappings(final String requestMappings, final ClusterState currentState, final String templateName, final NamedXContentRegistry xContentRegistry, final String indexName) throws Exception { - List> result = new ArrayList<>(); - List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName, xContentRegistry); + return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); + } + + public static List> collectV2Mappings(final String requestMappings, + final List templateMappings, + final NamedXContentRegistry xContentRegistry) throws Exception { + List> result = new ArrayList<>(); + for (CompressedXContent templateMapping : templateMappings) { Map parsedTemplateMapping = MapperService.parseMapping(xContentRegistry, templateMapping.string()); result.add(parsedTemplateMapping); @@ -533,6 +596,7 @@ public static List> collectV2Mappings(final String requestMa result.add(parsedRequestMappings); return result; } + private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterState currentState, final CreateIndexClusterStateUpdateRequest request, final boolean silent, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 2cde3491fedb0..619a843b7f25b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -432,7 +432,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS public static void validateV2TemplateRequest(Metadata metadata, String name, ComposableIndexTemplate template) { if (template.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) { - Settings mergedSettings = resolveSettings(metadata, template); + Settings mergedSettings = resolveSettings(template, metadata.componentTemplates()); if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(mergedSettings)) { throw new InvalidIndexTemplateException(name, "global composable templates may not specify the setting " + IndexMetadata.INDEX_HIDDEN_SETTING.getKey()); @@ -998,6 +998,17 @@ public static List collectMappings(final ClusterState state, } final Map componentTemplates = state.metadata().componentTemplates(); + return collectMappings(template, componentTemplates, indexName, xContentRegistry); + } + + /** + * Collect the given v2 template into an ordered list of mappings. + */ + public static List collectMappings(final ComposableIndexTemplate template, + final Map componentTemplates, + final String indexName, + final NamedXContentRegistry xContentRegistry) throws Exception { + Objects.requireNonNull(template, "Composable index template must be provided"); List mappings = template.composedOf().stream() .map(componentTemplates::get) .filter(Objects::nonNull) @@ -1057,11 +1068,15 @@ public static Settings resolveSettings(final Metadata metadata, final String tem if (template == null) { return Settings.EMPTY; } - return resolveSettings(metadata, template); + return resolveSettings(template, metadata.componentTemplates()); } - private static Settings resolveSettings(Metadata metadata, ComposableIndexTemplate template) { - final Map componentTemplates = metadata.componentTemplates(); + /** + * Resolve the provided v2 template and component templates into a collected {@link Settings} object + */ + static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { + Objects.requireNonNull(template, "attempted to resolve settings for a null template"); + Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); List componentSettings = template.composedOf().stream() .map(componentTemplates::get) .filter(Objects::nonNull) @@ -1107,6 +1122,17 @@ public static List> resolveAliases(final Metadata met return List.of(); } final Map componentTemplates = metadata.componentTemplates(); + return resolveAliases(template, componentTemplates, templateName); + } + + /** + * Resolve the given v2 template and component templates into an ordered list of aliases + */ + static List> resolveAliases(final ComposableIndexTemplate template, + final Map componentTemplates, + @Nullable String templateName) { + Objects.requireNonNull(template, "attempted to resolve aliases for a null template"); + Objects.requireNonNull(componentTemplates, "attempted to resolve aliases with null component templates"); List> aliases = template.composedOf().stream() .map(componentTemplates::get) .filter(Objects::nonNull) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index ba443d9364fd6..a34b2e906df74 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -20,9 +20,9 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesService; @@ -43,14 +43,18 @@ public class MetadataMigrateToDataStreamService { private final ClusterService clusterService; private final ActiveShardsObserver activeShardsObserver; private final IndicesService indexServices; + private final ThreadContext threadContext; + private final MetadataCreateIndexService metadataCreateIndexService; - @Inject public MetadataMigrateToDataStreamService(ThreadPool threadPool, ClusterService clusterService, - IndicesService indexServices) { + IndicesService indexServices, + MetadataCreateIndexService metadataCreateIndexService) { this.clusterService = clusterService; this.indexServices = indexServices; this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); + this.threadContext = threadPool.getThreadContext(); + this.metadataCreateIndexService = metadataCreateIndexService; } public void migrateToDataStream(MigrateToDataStreamClusterStateUpdateRequest request, @@ -89,7 +93,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { throw new IllegalStateException(e); } }, - request); + request, + threadContext, + metadataCreateIndexService); writeIndexRef.set(clusterState.metadata().dataStreams().get(request.aliasName).getWriteIndex().getName()); return clusterState; } @@ -98,7 +104,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { static ClusterState migrateToDataStream(ClusterState currentState, Function mapperSupplier, - MigrateToDataStreamClusterStateUpdateRequest request) throws Exception { + MigrateToDataStreamClusterStateUpdateRequest request, + ThreadContext threadContext, + MetadataCreateIndexService metadataCreateIndexService) throws Exception { validateRequest(currentState, request); IndexAbstraction.Alias alias = (IndexAbstraction.Alias) currentState.metadata().getIndicesLookup().get(request.aliasName); @@ -117,7 +125,8 @@ static ClusterState migrateToDataStream(ClusterState currentState, .collect(Collectors.toList()); logger.info("submitting request to migrate alias [{}] to a data stream", request.aliasName); - return MetadataCreateDataStreamService.createDataStream(null, currentState, request.aliasName, backingIndices, writeIndex); + return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, currentState, request.aliasName, + backingIndices, writeIndex, threadContext); } // package-visible for testing diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java index dfce857d69099..1e0144fe3e39d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java @@ -54,7 +54,9 @@ public void clusterChanged(ClusterChangedEvent event) { if (lastIndexMetadataMap != indexMetadataMap) { for (ObjectObjectCursor cursor : indexMetadataMap) { if (cursor.value != lastIndexMetadataMap.get(cursor.key)) { - if (systemIndices.isSystemIndex(cursor.value.getIndex()) != cursor.value.isSystem()) { + final boolean isSystem = systemIndices.isSystemIndex(cursor.value.getIndex()) || + systemIndices.isSystemIndexBackingDataStream(cursor.value.getIndex().getName()); + if (isSystem != cursor.value.isSystem()) { updateTaskPending = true; clusterService.submitStateUpdateTask("system_index_metadata_upgrade_service {system metadata change}", new SystemIndexMetadataUpdateTask()); @@ -74,7 +76,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { final List updatedMetadata = new ArrayList<>(); for (ObjectObjectCursor cursor : indexMetadataMap) { if (cursor.value != lastIndexMetadataMap.get(cursor.key)) { - if (systemIndices.isSystemIndex(cursor.value.getIndex()) != cursor.value.isSystem()) { + final boolean isSystem = systemIndices.isSystemIndex(cursor.value.getIndex()) || + systemIndices.isSystemIndexBackingDataStream(cursor.value.getIndex().getName()); + if (isSystem != cursor.value.isSystem()) { updatedMetadata.add(IndexMetadata.builder(cursor.value).system(cursor.value.isSystem() == false).build()); } } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java new file mode 100644 index 0000000000000..66fc557496667 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices; + +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; + +import java.util.List; +import java.util.Map; + +public class SystemDataStreamDescriptor { + + private final String dataStreamName; + private final String description; + private final Type type; + private final ComposableIndexTemplate composableIndexTemplate; + private final Map componentTemplates; + private final List allowedElasticProductOrigins; + + public SystemDataStreamDescriptor(String dataStreamName, String description, Type type, + ComposableIndexTemplate composableIndexTemplate, Map componentTemplates, + List allowedElasticProductOrigins) { + // TODO validation and javadocs + this.dataStreamName = dataStreamName; + this.description = description; + this.type = type; + this.composableIndexTemplate = composableIndexTemplate; + this.componentTemplates = Map.copyOf(componentTemplates); + this.allowedElasticProductOrigins = allowedElasticProductOrigins; + } + + public String getDataStreamName() { + return dataStreamName; + } + + public String getDescription() { + return description; + } + + public ComposableIndexTemplate getComposableIndexTemplate() { + return composableIndexTemplate; + } + + public boolean isExternal() { + return type == Type.EXTERNAL; + } + + public String getBackingIndexPattern() { + return DataStream.BACKING_INDEX_PREFIX + getDataStreamName() + "-*"; + } + + public List getAllowedElasticProductOrigins() { + return allowedElasticProductOrigins; + } + + public Map getComponentTemplates() { + return componentTemplates; + } + + public enum Type { + INTERNAL, + EXTERNAL + } +} diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index f1a5c852c6bfb..67205ed4a6f13 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -21,10 +21,12 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.Index; import org.elasticsearch.snapshots.SnapshotsService; @@ -36,6 +38,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -50,11 +53,18 @@ * to reduce the locations within the code that need to deal with {@link SystemIndexDescriptor}s. */ public class SystemIndices { + public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed"; + public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin"; + + private static final Automaton EMPTY = Automata.makeEmpty(); + private static final Map SERVER_SYSTEM_INDEX_DESCRIPTORS = Map.of( TASKS_FEATURE_NAME, new Feature(TASKS_FEATURE_NAME, "Manages task results", List.of(TASKS_DESCRIPTOR)) ); - private final CharacterRunAutomaton runAutomaton; + private final CharacterRunAutomaton systemIndexAutomaton; + private final CharacterRunAutomaton systemDataStreamIndicesAutomaton; + private final Predicate systemDataStreamAutomaton; private final Map featureDescriptors; private final Map productToSystemIndicesMatcher; @@ -67,11 +77,13 @@ public SystemIndices(Map pluginAndModulesDescriptors) { featureDescriptors = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); checkForOverlappingPatterns(featureDescriptors); checkForDuplicateAliases(this.getSystemIndexDescriptors()); - this.runAutomaton = buildCharacterRunAutomaton(featureDescriptors); - this.productToSystemIndicesMatcher = getProductToSystemIndicesMap(this.getSystemIndexDescriptors()); + this.systemIndexAutomaton = buildIndexCharacterRunAutomaton(featureDescriptors); + this.systemDataStreamIndicesAutomaton = buildDataStreamBackingIndicesAutomaton(featureDescriptors); + this.systemDataStreamAutomaton = buildDataStreamNamePredicate(featureDescriptors); + this.productToSystemIndicesMatcher = getProductToSystemIndicesMap(featureDescriptors); } - private void checkForDuplicateAliases(Collection descriptors) { + private static void checkForDuplicateAliases(Collection descriptors) { final Map aliasCounts = new HashMap<>(); for (SystemIndexDescriptor descriptor : descriptors) { @@ -93,20 +105,42 @@ private void checkForDuplicateAliases(Collection descript } } - private static Map getProductToSystemIndicesMap(Collection descriptors) { - Map map = descriptors.stream() - .filter(SystemIndexDescriptor::isExternal) - .flatMap(descriptor -> descriptor.getAllowedElasticProductOrigins().stream().map(product -> new Tuple<>(product, descriptor))) - .collect(Collectors.toUnmodifiableMap(Tuple::v1, tuple -> { - SystemIndexDescriptor descriptor = tuple.v2(); - return SystemIndexDescriptor.buildAutomaton(descriptor.getIndexPattern(), descriptor.getAliasName()); - }, Operations::union)); + private static Map getProductToSystemIndicesMap(Map descriptors) { + Map productToSystemIndicesMap = new HashMap<>(); + for (Feature feature : descriptors.values()) { + feature.getIndexDescriptors().forEach(systemIndexDescriptor -> { + if (systemIndexDescriptor.isExternal()) { + systemIndexDescriptor.getAllowedElasticProductOrigins().forEach(origin -> + productToSystemIndicesMap.compute(origin, (key, value) -> { + Automaton automaton = SystemIndexDescriptor.buildAutomaton( + systemIndexDescriptor.getIndexPattern(), systemIndexDescriptor.getAliasName()); + return value == null ? automaton : Operations.union(value, automaton); + }) + ); + } + }); + feature.getDataStreamDescriptors().forEach(dataStreamDescriptor -> { + if (dataStreamDescriptor.isExternal()) { + dataStreamDescriptor.getAllowedElasticProductOrigins().forEach(origin -> + productToSystemIndicesMap.compute(origin, (key, value) -> { + Automaton automaton = SystemIndexDescriptor.buildAutomaton( + dataStreamDescriptor.getBackingIndexPattern(), dataStreamDescriptor.getDataStreamName()); + return value == null ? automaton : Operations.union(value, automaton); + }) + ); + } + }); + } - return map.entrySet().stream() + return productToSystemIndicesMap.entrySet().stream() .collect(Collectors.toUnmodifiableMap(Entry::getKey, entry -> new CharacterRunAutomaton(MinimizationOperations.minimize(entry.getValue(), Integer.MAX_VALUE)))); } + public boolean isSystemName(String name) { + return isSystemIndex(name) || isSystemDataStream(name) || isSystemIndexBackingDataStream(name); + } + /** * Determines whether a given index is a system index by comparing its name to the collection of loaded {@link SystemIndexDescriptor}s * @param index the {@link Index} object to check against loaded {@link SystemIndexDescriptor}s @@ -122,7 +156,15 @@ public boolean isSystemIndex(Index index) { * @return true if the index name matches a pattern from a {@link SystemIndexDescriptor} */ public boolean isSystemIndex(String indexName) { - return runAutomaton.run(indexName); + return systemIndexAutomaton.run(indexName); + } + + public boolean isSystemDataStream(String name) { + return systemDataStreamAutomaton.test(name); + } + + public boolean isSystemIndexBackingDataStream(String name) { + return systemDataStreamIndicesAutomaton.run(name); } /** @@ -156,12 +198,48 @@ public boolean isSystemIndex(String indexName) { } } + /** + * Finds a single matching {@link SystemDataStreamDescriptor}, if any, for the given DataStream name. + * @param name the name of the DataStream + * @return The matching {@link SystemDataStreamDescriptor} or {@code null} if no descriptor is found + * @throws IllegalStateException if multiple descriptors match the name + */ + public @Nullable SystemDataStreamDescriptor findMatchingDataStreamDescriptor(String name) { + final List matchingDescriptors = featureDescriptors.values().stream() + .flatMap(feature -> feature.getDataStreamDescriptors().stream()) + .filter(descriptor -> descriptor.getDataStreamName().equals(name)) + .collect(toUnmodifiableList()); + + if (matchingDescriptors.isEmpty()) { + return null; + } else if (matchingDescriptors.size() == 1) { + return matchingDescriptors.get(0); + } else { + // This should be prevented by failing on overlapping patterns at startup time, but is here just in case. + StringBuilder errorMessage = new StringBuilder() + .append("DataStream name [") + .append(name) + .append("] is claimed as a system data stream by multiple descriptors: [") + .append(matchingDescriptors.stream() + .map(descriptor -> "name: [" + descriptor.getDataStreamName() + + "], description: [" + descriptor.getDescription() + "]").collect(Collectors.joining("; "))); + // Throw AssertionError if assertions are enabled, or a regular exception otherwise: + assert false : errorMessage.toString(); + throw new IllegalStateException(errorMessage.toString()); + } + } + /** * Builds a predicate that tests if a system index should be accessible based on the provided product name - * @param product the name of the product that is attempting to access an external system index + * contained in headers. + * @param threadContext the threadContext containing headers used for system index access * @return Predicate to check external system index metadata with */ - public Predicate getProductSystemIndexMetadataPredicate(String product) { + public Predicate getProductSystemIndexMetadataPredicate(ThreadContext threadContext) { + final String product = threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY); + if (product == null) { + return indexMetadata -> false; + } final CharacterRunAutomaton automaton = productToSystemIndicesMatcher.get(product); if (automaton == null) { return indexMetadata -> false; @@ -171,10 +249,15 @@ public Predicate getProductSystemIndexMetadataPredicate(String pr /** * Builds a predicate that tests if a system index name should be accessible based on the provided product name - * @param product the name of the product that is attempting to access an external system index + * contained in headers. + * @param threadContext the threadContext containing headers used for system index access * @return Predicate to check external system index names with */ - public Predicate getProductSystemIndexNamePredicate(String product) { + public Predicate getProductSystemIndexNamePredicate(ThreadContext threadContext) { + final String product = threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY); + if (product == null) { + return name -> false; + } final CharacterRunAutomaton automaton = productToSystemIndicesMatcher.get(product); if (automaton == null) { return name -> false; @@ -186,12 +269,103 @@ public Map getFeatures() { return featureDescriptors; } - private static CharacterRunAutomaton buildCharacterRunAutomaton(Map descriptors) { + private static CharacterRunAutomaton buildIndexCharacterRunAutomaton(Map descriptors) { Optional automaton = descriptors.values().stream() - .flatMap(feature -> feature.getIndexDescriptors().stream()) + .map(SystemIndices::featureToIndexAutomaton) + .reduce(Operations::union); + return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE)); + } + + private static Automaton featureToIndexAutomaton(Feature feature) { + Optional systemIndexAutomaton = feature.getIndexDescriptors().stream() .map(descriptor -> SystemIndexDescriptor.buildAutomaton(descriptor.getIndexPattern(), descriptor.getAliasName())) .reduce(Operations::union); - return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(Automata.makeEmpty()), Integer.MAX_VALUE)); + + return systemIndexAutomaton.orElse(EMPTY); + } + + private static Predicate buildDataStreamNamePredicate(Map descriptors) { + Set systemDataStreamNames = descriptors.values().stream() + .flatMap(feature -> feature.getDataStreamDescriptors().stream()) + .map(SystemDataStreamDescriptor::getDataStreamName) + .collect(Collectors.toUnmodifiableSet()); + return systemDataStreamNames::contains; + } + + private static CharacterRunAutomaton buildDataStreamBackingIndicesAutomaton(Map descriptors) { + Optional automaton = descriptors.values().stream() + .map(SystemIndices::featureToDataStreamBackingIndicesAutomaton) + .reduce(Operations::union); + return new CharacterRunAutomaton(automaton.orElse(EMPTY)); + } + + private static Automaton featureToDataStreamBackingIndicesAutomaton(Feature feature) { + Optional systemDataStreamAutomaton = feature.getDataStreamDescriptors().stream() + .map(descriptor -> SystemIndexDescriptor.buildAutomaton( + descriptor.getBackingIndexPattern(), + null + )) + .reduce(Operations::union); + return systemDataStreamAutomaton.orElse(EMPTY); + } + + public SystemDataStreamDescriptor validateDataStreamAccess(String dataStreamName, ThreadContext threadContext) { + if (systemDataStreamAutomaton.test(dataStreamName)) { + SystemDataStreamDescriptor dataStreamDescriptor = featureDescriptors.values().stream() + .flatMap(feature -> feature.getDataStreamDescriptors().stream()) + .filter(descriptor -> descriptor.getDataStreamName().equals(dataStreamName)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("system datastream descriptor not found for [" + dataStreamName + "]")); + if (dataStreamDescriptor.isExternal()) { + final SystemIndexAccessLevel accessLevel = getSystemIndexAccessLevel(threadContext); + if (accessLevel == SystemIndexAccessLevel.NONE) { + throw new IllegalArgumentException("DataStream [" + dataStreamName + "] is reserved for system use and access"); + } else if (accessLevel == SystemIndexAccessLevel.RESTRICTED) { + if (getProductSystemIndexNamePredicate(threadContext).test(dataStreamName) == false) { + throw new IllegalArgumentException("DataStream [" + dataStreamName + "] is not accessible by product [" + + threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY) + "]"); + } else { + return dataStreamDescriptor; + } + } else { + assert accessLevel == SystemIndexAccessLevel.ALL; + return dataStreamDescriptor; + } + } else { + return dataStreamDescriptor; + } + } else { + return null; + } + } + + /** + * Determines what level of system index access should be allowed in the current context. + * + * @return {@link SystemIndexAccessLevel#ALL} if unrestricted system index access should be allowed, + * {@link SystemIndexAccessLevel#RESTRICTED} if a subset of system index access should be allowed, or + * {@link SystemIndexAccessLevel#NONE} if no system index access should be allowed. + */ + public SystemIndexAccessLevel getSystemIndexAccessLevel(ThreadContext threadContext) { + final String headerValue = threadContext.getHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY); + final String productHeaderValue = threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY); + + final boolean allowed = Booleans.parseBoolean(headerValue, true); + if (allowed) { + if (productHeaderValue != null) { + return SystemIndexAccessLevel.RESTRICTED; + } else { + return SystemIndexAccessLevel.ALL; + } + } else { + return SystemIndexAccessLevel.NONE; + } + } + + public enum SystemIndexAccessLevel { + ALL, + NONE, + RESTRICTED } /** @@ -224,6 +398,7 @@ static void checkForOverlappingPatterns(Map sourceToDescriptors .collect(Collectors.joining(", "))); } }); + // TODO this needs to be re-worked to account for alias, primaryIndex, and datastreams } private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) { @@ -269,6 +444,7 @@ public static void validateFeatureName(String name, String plugin) { public static class Feature { private final String description; private final Collection indexDescriptors; + private final Collection dataStreamDescriptors; private final Collection associatedIndexPatterns; private final TriConsumer> cleanUpFunction; @@ -282,10 +458,12 @@ public static class Feature { public Feature( String description, Collection indexDescriptors, + Collection dataStreamDescriptors, Collection associatedIndexPatterns, TriConsumer> cleanUpFunction) { this.description = description; this.indexDescriptors = indexDescriptors; + this.dataStreamDescriptors = dataStreamDescriptors; this.associatedIndexPatterns = associatedIndexPatterns; this.cleanUpFunction = cleanUpFunction; } @@ -297,7 +475,7 @@ public Feature( * @param indexDescriptors Patterns describing system indices for this feature */ public Feature(String name, String description, Collection indexDescriptors) { - this(description, indexDescriptors, Collections.emptyList(), + this(description, indexDescriptors, Collections.emptyList(), Collections.emptyList(), (clusterService, client, listener) -> cleanUpFeature(indexDescriptors, Collections.emptyList(), name, clusterService, client, listener) ); @@ -311,6 +489,10 @@ public Collection getIndexDescriptors() { return indexDescriptors; } + public Collection getDataStreamDescriptors() { + return dataStreamDescriptors; + } + public Collection getAssociatedIndexPatterns() { return associatedIndexPatterns; } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 67e06c016fe7a..24a0df70503ed 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -402,6 +402,26 @@ protected Node(final Environment initialEnvironment, final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client); final UsageService usageService = new UsageService(); + SearchModule searchModule = new SearchModule(settings, pluginsService.filterPlugins(SearchPlugin.class)); + List namedWriteables = Stream.of( + NetworkModule.getNamedWriteables().stream(), + IndicesModule.getNamedWriteables().stream(), + searchModule.getNamedWriteables().stream(), + pluginsService.filterPlugins(Plugin.class).stream() + .flatMap(p -> p.getNamedWriteables().stream()), + ClusterModule.getNamedWriteables().stream()) + .flatMap(Function.identity()).collect(Collectors.toList()); + final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + searchModule.getNamedXContents().stream(), + pluginsService.filterPlugins(Plugin.class).stream() + .flatMap(p -> p.getNamedXContent().stream()), + ClusterModule.getNamedXWriteables().stream()) + .flatMap(Function.identity()).collect(toList()), + getCompatibleNamedXContents() + ); final Map featuresMap = pluginsService .filterPlugins(SystemIndexPlugin.class) .stream() @@ -411,6 +431,7 @@ protected Node(final Environment initialEnvironment, plugin -> new SystemIndices.Feature( plugin.getFeatureDescription(), plugin.getSystemIndexDescriptors(settings), + plugin.getSystemDataStreamDescriptors(), plugin.getAssociatedIndexPatterns(), plugin::cleanUpFeature )) @@ -430,7 +451,6 @@ protected Node(final Environment initialEnvironment, IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class)); modules.add(indicesModule); - SearchModule searchModule = new SearchModule(settings, pluginsService.filterPlugins(SearchPlugin.class)); List pluginCircuitBreakers = pluginsService.filterPlugins(CircuitBreakerPlugin.class) .stream() .map(plugin -> plugin.getCircuitBreaker(settings)) @@ -450,25 +470,6 @@ protected Node(final Environment initialEnvironment, PageCacheRecycler pageCacheRecycler = createPageCacheRecycler(settings); BigArrays bigArrays = createBigArrays(pageCacheRecycler, circuitBreakerService); modules.add(settingsModule); - List namedWriteables = Stream.of( - NetworkModule.getNamedWriteables().stream(), - IndicesModule.getNamedWriteables().stream(), - searchModule.getNamedWriteables().stream(), - pluginsService.filterPlugins(Plugin.class).stream() - .flatMap(p -> p.getNamedWriteables().stream()), - ClusterModule.getNamedWriteables().stream()) - .flatMap(Function.identity()).collect(Collectors.toList()); - final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); - NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of( - NetworkModule.getNamedXContents().stream(), - IndicesModule.getNamedXContents().stream(), - searchModule.getNamedXContents().stream(), - pluginsService.filterPlugins(Plugin.class).stream() - .flatMap(p -> p.getNamedXContent().stream()), - ClusterModule.getNamedXWriteables().stream()) - .flatMap(Function.identity()).collect(toList()), - getCompatibleNamedXContents() - ); final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry); final PersistedClusterStateService lucenePersistedStateFactory = new PersistedClusterStateService(nodeEnvironment, xContentRegistry, bigArrays, clusterService.getClusterSettings(), diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java index 9724b7eeb182c..26b1fae45843e 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; @@ -35,6 +36,10 @@ default Collection getSystemIndexDescriptors(Settings set return Collections.emptyList(); } + default Collection getSystemDataStreamDescriptors() { + return Collections.emptyList(); + } + /** * @return The name of the feature, as used for specifying feature states in snapshot creation and restoration. */ diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index ae788e9101c2a..2cd94fcfad893 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -44,8 +44,8 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; +import static org.elasticsearch.indices.SystemIndices.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; +import static org.elasticsearch.indices.SystemIndices.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; import static org.elasticsearch.rest.BytesRestResponse.TEXT_CONTENT_TYPE; import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 926dc140b7251..f7e97735ad8f0 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -755,7 +755,7 @@ public void onFailure(Exception e) { } private boolean isSystemIndex(IndexMetadata indexMetadata) { - return indexMetadata.isSystem() || systemIndices.isSystemIndex(indexMetadata.getIndex()); + return indexMetadata.isSystem() || systemIndices.isSystemName(indexMetadata.getIndex().getName()); } private Map getDataStreamsToRestore(Repository repository, SnapshotId snapshotId, SnapshotInfo snapshotInfo, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index 7630859173d65..f8565133bb571 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -11,12 +11,14 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SystemIndexAccessLevel; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import org.elasticsearch.test.ESTestCase; import java.util.Collections; @@ -130,7 +132,7 @@ public void testDeprecationWarningNotEmittedWhenSystemAccessAllowed() { assertEquals(state.metadata().findAliases(request, concreteIndices), aliases); ImmutableOpenMap> result = TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, - SystemIndexAccessLevel.ALL, "", EmptySystemIndices.INSTANCE); + SystemIndexAccessLevel.ALL, new ThreadContext(Settings.EMPTY), EmptySystemIndices.INSTANCE); assertThat(result.size(), equalTo(1)); assertThat(result.get(".b").size(), equalTo(1)); } @@ -150,7 +152,7 @@ public void testDeprecationWarningNotEmittedWhenOnlyNonsystemIndexRequested() { assertEquals(state.metadata().findAliases(request, concreteIndices), aliases); ImmutableOpenMap> result = TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, - SystemIndexAccessLevel.NONE, "", EmptySystemIndices.INSTANCE); + SystemIndexAccessLevel.NONE, new ThreadContext(Settings.EMPTY), EmptySystemIndices.INSTANCE); assertThat(result.size(), equalTo(1)); assertThat(result.get("c").size(), equalTo(1)); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java index 09045500a877d..5b74e46b0f671 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.DateMathExpressionResolver; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SystemIndexAccessLevel; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -25,7 +24,7 @@ import java.util.Collections; import java.util.List; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SystemIndexAccessLevel.NONE; +import static org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel.NONE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.joda.time.DateTimeZone.UTC; @@ -35,7 +34,7 @@ public class DateMathExpressionResolverTests extends ESTestCase { private final DateMathExpressionResolver expressionResolver = new DateMathExpressionResolver(); private final Context context = new Context( ClusterState.builder(new ClusterName("_name")).build(), IndicesOptions.strictExpand(), - SystemIndexAccessLevel.NONE + NONE ); public void testNormal() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index c7f62c4d44fbb..3b52bbafc93d7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -54,10 +54,10 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_HIDDEN_SETTING; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SystemIndexAccessLevel.NONE; import static org.elasticsearch.common.util.set.Sets.newHashSet; +import static org.elasticsearch.indices.SystemIndices.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; +import static org.elasticsearch.indices.SystemIndices.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; +import static org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel.NONE; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.arrayWithSize; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index b32ff87fcc221..7bee3aa8d7515 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -15,6 +15,8 @@ import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.test.ESTestCase; import java.util.List; @@ -45,7 +47,8 @@ public void testCreateDataStream() throws Exception { .build(); CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); - ClusterState newState = MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); + ClusterState newState = + MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY)); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue()); @@ -65,7 +68,7 @@ public void testCreateDuplicateDataStream() throws Exception { new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] already exists")); } @@ -76,7 +79,7 @@ public void testCreateDataStreamWithInvalidName() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); assertThat(e.getMessage(), containsString("must not contain the following characters")); } @@ -87,7 +90,7 @@ public void testCreateDataStreamWithUppercaseCharacters() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must be lowercase")); } @@ -98,7 +101,7 @@ public void testCreateDataStreamStartingWithPeriod() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.ds-'")); } @@ -110,7 +113,7 @@ public void testCreateDataStreamNoTemplate() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); Exception e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); assertThat(e.getMessage(), equalTo("no matching index template found for data stream [my-data-stream]")); } @@ -125,7 +128,7 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); Exception e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); assertThat(e.getMessage(), equalTo("matching index template [template] for data stream [my-data-stream] has no data stream template")); } @@ -141,11 +144,12 @@ public static ClusterState createDataStream(final String dataStreamName) throws .build(); MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest req = new MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); - return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); + return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY)); } private static MetadataCreateIndexService getMetadataCreateIndexService() throws Exception { MetadataCreateIndexService s = mock(MetadataCreateIndexService.class); + when(s.getSystemIndices()).thenReturn(EmptySystemIndices.INSTANCE); when(s.applyCreateIndexRequest(any(ClusterState.class), any(CreateIndexClusterStateUpdateRequest.class), anyBoolean())) .thenAnswer(mockInvocation -> { ClusterState currentState = (ClusterState) mockInvocation.getArguments()[0]; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index 772b42d094799..fd76706fd591d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -14,8 +14,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.plugins.Plugin; import java.io.IOException; @@ -29,6 +31,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MetadataMigrateToDataStreamServiceTests extends MapperServiceTestCase { @@ -207,7 +211,10 @@ public void testCreateDataStreamWithSuppliedWriteIndex() throws Exception { ClusterState newState = MetadataMigrateToDataStreamService.migrateToDataStream(cs, this::getMapperService, new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, - TimeValue.ZERO)); + TimeValue.ZERO), + new ThreadContext(Settings.EMPTY), + getMetadataCreateIndexService() + ); IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); assertThat(ds, notNullValue()); assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); @@ -249,7 +256,10 @@ public void testCreateDataStreamHidesBackingIndicesAndRemovesAlias() throws Exce ClusterState newState = MetadataMigrateToDataStreamService.migrateToDataStream(cs, this::getMapperService, new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, - TimeValue.ZERO)); + TimeValue.ZERO), + new ThreadContext(Settings.EMPTY), + getMetadataCreateIndexService() + ); IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); assertThat(ds, notNullValue()); assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); @@ -298,7 +308,9 @@ public void testCreateDataStreamWithoutSuppliedWriteIndex() throws Exception { this::getMapperService, new MetadataMigrateToDataStreamService.MigrateToDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, - TimeValue.ZERO))); + TimeValue.ZERO), + new ThreadContext(Settings.EMPTY), + getMetadataCreateIndexService())); assertThat(e.getMessage(), containsString("alias [" + dataStreamName + "] must specify a write index")); } @@ -310,6 +322,12 @@ private MapperService getMapperService(IndexMetadata im) { } } + private MetadataCreateIndexService getMetadataCreateIndexService() { + MetadataCreateIndexService service = mock(MetadataCreateIndexService.class); + when(service.getSystemIndices()).thenReturn(EmptySystemIndices.INSTANCE); + return service; + } + @Override protected Collection getPlugins() { return List.of(new MetadataIndexTemplateServiceTests.DummyPlugin()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java index d3825b99c7929..5ab3104491a66 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -23,8 +23,8 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SystemIndexAccessLevel.NONE; import static org.elasticsearch.common.util.set.Sets.newHashSet; +import static org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel.NONE; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index a5a00fe15bd8b..2a67f0e739df6 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -11,9 +11,6 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; @@ -137,7 +134,7 @@ public static DataStream randomInstance(LongSupplier timeProvider) { metadata = Map.of("key", "value"); } return new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation, metadata, - randomBoolean(), randomBoolean(), timeProvider); + randomBoolean(), randomBoolean(), false, timeProvider); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java index 1ab0fa5577560..d78097475603f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java @@ -13,12 +13,12 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SystemIndexAccessLevel; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import java.util.Collections; import java.util.List; diff --git a/x-pack/plugin/core/src/main/resources/fleet-actions-results.json b/x-pack/plugin/core/src/main/resources/fleet-actions-results.json new file mode 100644 index 0000000000000..eb284e788b70c --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/fleet-actions-results.json @@ -0,0 +1,54 @@ +{ + "index_patterns": [ + ".fleet-actions-results" + ], + "data_stream": {}, + "template": { + "settings": { + "index.lifecycle.name": ".fleet-actions-results-ilm-policy" + }, + "mappings": { + "_meta": { + "version": "${fleet.version}" + }, + "dynamic": false, + "properties": { + "action_id": { + "type": "keyword" + }, + "agent_id": { + "type": "keyword" + }, + "action_data": { + "enabled": false, + "type": "object" + }, + "data": { + "enabled": false, + "type": "object" + }, + "error": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "@timestamp": { + "type": "date" + }, + "started_at": { + "type": "date" + }, + "completed_at": { + "type": "date" + } + } + } + }, + "composed_of": [], + "priority": 200, + "version": 1 +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java index 273f8f55a3b22..5d4d48703172c 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java @@ -17,6 +17,8 @@ import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -25,6 +27,7 @@ public class CreateDataStreamTransportAction extends AcknowledgedTransportMasterNodeAction { private final MetadataCreateDataStreamService metadataCreateDataStreamService; + private final SystemIndices systemIndices; @Inject public CreateDataStreamTransportAction( @@ -33,7 +36,8 @@ public CreateDataStreamTransportAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - MetadataCreateDataStreamService metadataCreateDataStreamService + MetadataCreateDataStreamService metadataCreateDataStreamService, + SystemIndices systemIndices ) { super( CreateDataStreamAction.NAME, @@ -46,6 +50,7 @@ public CreateDataStreamTransportAction( ThreadPool.Names.SAME ); this.metadataCreateDataStreamService = metadataCreateDataStreamService; + this.systemIndices = systemIndices; } @Override @@ -55,9 +60,12 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { + final SystemDataStreamDescriptor systemDataStreamDescriptor = + systemIndices.validateDataStreamAccess(request.getName(), threadPool.getThreadContext()); MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest updateRequest = new MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest( request.getName(), + systemDataStreamDescriptor, request.masterNodeTimeout(), request.timeout() ); diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java index 64db3d77d8485..1ddf4885afd96 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java @@ -15,9 +15,11 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataMigrateToDataStreamService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -34,7 +36,8 @@ public MigrateToDataStreamTransportAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - MetadataMigrateToDataStreamService metadataMigrateToDataStreamService + IndicesService indicesService, + MetadataCreateIndexService metadataCreateIndexService ) { super( MigrateToDataStreamAction.NAME, @@ -46,7 +49,8 @@ public MigrateToDataStreamTransportAction( indexNameExpressionResolver, ThreadPool.Names.SAME ); - this.metadataMigrateToDataStreamService = metadataMigrateToDataStreamService; + this.metadataMigrateToDataStreamService = + new MetadataMigrateToDataStreamService(threadPool, clusterService, indicesService, metadataCreateIndexService); } @Override diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index ea9fde1daca59..89ddc73b96135 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -171,4 +171,11 @@ public void testCreationOfFleetServers() throws Exception { responseBody = EntityUtils.toString(response.getEntity()); assertThat(responseBody, containsString("architecture")); } + + public void testCreationOfFleetActionsResults() throws Exception { + Request request = new Request("POST", "/.fleet-actions-results/_doc"); + request.setJsonEntity("{ \"@timestamp\": \"2099-03-08T11:06:07.000Z\", \"agent_id\": \"my-agent\" }"); + Response response = client().performRequest(request); + assertEquals(201, response.getStatusLine().getStatusCode()); + } } diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index edc97d947451d..8f38511c99f25 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -9,16 +9,24 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor.Type; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Collection; import java.util.List; +import java.util.Map; import static org.elasticsearch.xpack.core.ClientHelper.FLEET_ORIGIN; @@ -47,6 +55,11 @@ public Collection getSystemIndexDescriptors(Settings sett ); } + @Override + public Collection getSystemDataStreamDescriptors() { + return List.of(fleetActionsResultsDescriptor()); + } + @Override public String getFeatureName() { return "fleet"; @@ -183,6 +196,24 @@ private SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { .build(); } + private SystemDataStreamDescriptor fleetActionsResultsDescriptor() { + final String source = loadTemplateSource("/fleet-actions-results.json"); + try (XContentParser parser = XContentType.JSON.xContent().createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, source)) { + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.parse(parser); + return new SystemDataStreamDescriptor( + ".fleet-actions-results", + "Result history of fleet actions", + SystemDataStreamDescriptor.Type.EXTERNAL, + composableIndexTemplate, + Map.of(), + ALLOWED_PRODUCTS + ); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private String loadTemplateSource(String resource) { return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), MAPPING_VERSION_VARIABLE); } From d4b58f1d6e50f69798f1742701a2efb9cab1a62e Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 13 Apr 2021 21:02:51 -0600 Subject: [PATCH 02/22] spotless --- .../src/main/java/org/elasticsearch/xpack/fleet/Fleet.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 8f38511c99f25..92161f81f1d04 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -198,8 +198,10 @@ private SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { private SystemDataStreamDescriptor fleetActionsResultsDescriptor() { final String source = loadTemplateSource("/fleet-actions-results.json"); - try (XContentParser parser = XContentType.JSON.xContent().createParser( - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, source)) { + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, source) + ) { ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.parse(parser); return new SystemDataStreamDescriptor( ".fleet-actions-results", From 8cee7297794a37d73f97021b96269a5dbd9356f9 Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 13 Apr 2021 21:10:40 -0600 Subject: [PATCH 03/22] more spotless --- .../action/CreateDataStreamTransportAction.java | 6 ++++-- .../action/MigrateToDataStreamTransportAction.java | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java index 5d4d48703172c..e4618432a2602 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java @@ -60,8 +60,10 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { - final SystemDataStreamDescriptor systemDataStreamDescriptor = - systemIndices.validateDataStreamAccess(request.getName(), threadPool.getThreadContext()); + final SystemDataStreamDescriptor systemDataStreamDescriptor = systemIndices.validateDataStreamAccess( + request.getName(), + threadPool.getThreadContext() + ); MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest updateRequest = new MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest( request.getName(), diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java index 1ddf4885afd96..3493763aa2de1 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/MigrateToDataStreamTransportAction.java @@ -49,8 +49,12 @@ public MigrateToDataStreamTransportAction( indexNameExpressionResolver, ThreadPool.Names.SAME ); - this.metadataMigrateToDataStreamService = - new MetadataMigrateToDataStreamService(threadPool, clusterService, indicesService, metadataCreateIndexService); + this.metadataMigrateToDataStreamService = new MetadataMigrateToDataStreamService( + threadPool, + clusterService, + indicesService, + metadataCreateIndexService + ); } @Override From 6f242cb57498722555021e99a04cbe371db06d33 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 14 Apr 2021 10:02:25 -0600 Subject: [PATCH 04/22] add checks to other apis --- .../MetadataMigrateToDataStreamService.java | 1 + .../core/action/GetDataStreamAction.java | 2 + .../DeleteDataStreamTransportAction.java | 21 +++++++-- .../PromoteDataStreamTransportAction.java | 8 +++- .../rest/RestDataStreamsStatsAction.java | 5 +++ .../rest/RestGetDataStreamsAction.java | 5 +++ .../DeleteDataStreamTransportActionTests.java | 45 ++++++++++++++++--- 7 files changed, 77 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index a34b2e906df74..190a04bf16e63 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -59,6 +59,7 @@ public MetadataMigrateToDataStreamService(ThreadPool threadPool, public void migrateToDataStream(MigrateToDataStreamClusterStateUpdateRequest request, ActionListener finalListener) { + metadataCreateIndexService.getSystemIndices().validateDataStreamAccess(request.aliasName, threadContext); AtomicReference writeIndexRef = new AtomicReference<>(); ActionListener listener = ActionListener.wrap( response -> { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java index 12e69f1425ecd..c25db6213d4f6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java @@ -119,6 +119,7 @@ public static class DataStreamInfo extends AbstractDiffable impl public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template"); public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy"); public static final ParseField HIDDEN_FIELD = new ParseField("hidden"); + public static final ParseField SYSTEM_FIELD = new ParseField("system"); DataStream dataStream; ClusterHealthStatus dataStreamStatus; @@ -181,6 +182,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName); } builder.field(HIDDEN_FIELD.getPreferredName(), dataStream.isHidden()); + builder.field(SYSTEM_FIELD.getPreferredName(), dataStream.isSystem()); builder.endObject(); return builder; } diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportAction.java index 4de9513e30290..ace460a1e5d80 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.snapshots.SnapshotInProgressException; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.tasks.Task; @@ -38,6 +39,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Consumer; import static org.elasticsearch.xpack.datastreams.action.DataStreamsActionUtil.getDataStreamNames; @@ -46,6 +48,7 @@ public class DeleteDataStreamTransportAction extends AcknowledgedTransportMaster private static final Logger LOGGER = LogManager.getLogger(DeleteDataStreamTransportAction.class); private final MetadataDeleteIndexService deleteIndexService; + private final SystemIndices systemIndices; @Inject public DeleteDataStreamTransportAction( @@ -54,7 +57,8 @@ public DeleteDataStreamTransportAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - MetadataDeleteIndexService deleteIndexService + MetadataDeleteIndexService deleteIndexService, + SystemIndices systemIndices ) { super( DeleteDataStreamAction.NAME, @@ -67,6 +71,7 @@ public DeleteDataStreamTransportAction( ThreadPool.Names.SAME ); this.deleteIndexService = deleteIndexService; + this.systemIndices = systemIndices; } @Override @@ -87,7 +92,13 @@ public void onFailure(String source, Exception e) { @Override public ClusterState execute(ClusterState currentState) { - return removeDataStream(deleteIndexService, indexNameExpressionResolver, currentState, request); + return removeDataStream( + deleteIndexService, + indexNameExpressionResolver, + currentState, + request, + ds -> systemIndices.validateDataStreamAccess(ds, threadPool.getThreadContext()) + ); } @Override @@ -102,10 +113,14 @@ static ClusterState removeDataStream( MetadataDeleteIndexService deleteIndexService, IndexNameExpressionResolver indexNameExpressionResolver, ClusterState currentState, - DeleteDataStreamAction.Request request + DeleteDataStreamAction.Request request, + Consumer systemDataStreamAccessValidator ) { List names = getDataStreamNames(indexNameExpressionResolver, currentState, request.getNames(), request.indicesOptions()); Set dataStreams = new HashSet<>(names); + for (String dataStreamName : dataStreams) { + systemDataStreamAccessValidator.accept(dataStreamName); + } Set snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(currentState, dataStreams); if (dataStreams.isEmpty()) { diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/PromoteDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/PromoteDataStreamTransportAction.java index 435ffa8db459c..e8a025e2f7687 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/PromoteDataStreamTransportAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/PromoteDataStreamTransportAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -29,13 +30,16 @@ public class PromoteDataStreamTransportAction extends AcknowledgedTransportMasterNodeAction { + private final SystemIndices systemIndices; + @Inject public PromoteDataStreamTransportAction( TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + SystemIndices systemIndices ) { super( PromoteDataStreamAction.NAME, @@ -47,6 +51,7 @@ public PromoteDataStreamTransportAction( indexNameExpressionResolver, ThreadPool.Names.SAME ); + this.systemIndices = systemIndices; } @Override @@ -56,6 +61,7 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { + systemIndices.validateDataStreamAccess(request.getName(), threadPool.getThreadContext()); clusterService.submitStateUpdateTask( "promote-data-stream [" + request.getName() + "]", new ClusterStateUpdateTask(Priority.HIGH, request.masterNodeTimeout()) { diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDataStreamsStatsAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDataStreamsStatsAction.java index 080764c648028..3ed989f3f9685 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDataStreamsStatsAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDataStreamsStatsAction.java @@ -37,4 +37,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli dataStreamsStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, dataStreamsStatsRequest.indicesOptions())); return channel -> client.execute(DataStreamsStatsAction.INSTANCE, dataStreamsStatsRequest, new RestToXContentListener<>(channel)); } + + @Override + public boolean allowSystemIndexAccessByDefault() { + return true; + } } diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java index e544e01265506..a882ef112f17c 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java @@ -38,4 +38,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli getDataStreamsRequest.indicesOptions(IndicesOptions.fromRequest(request, getDataStreamsRequest.indicesOptions())); return channel -> client.execute(GetDataStreamAction.INSTANCE, getDataStreamsRequest, new RestToXContentListener<>(channel)); } + + @Override + public boolean allowSystemIndexAccessByDefault() { + return true; + } } diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java index 27ee8ea0afe0c..cc9aef7d7e417 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java @@ -9,16 +9,19 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; @@ -29,6 +32,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.function.Consumer; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -40,6 +44,10 @@ public class DeleteDataStreamTransportActionTests extends ESTestCase { private final IndexNameExpressionResolver iner = TestIndexNameExpressionResolver.newInstance(); + private final Consumer systemDataStreamValidator = ds -> EmptySystemIndices.INSTANCE.validateDataStreamAccess( + ds, + new ThreadContext(Settings.EMPTY) + ); public void testDeleteDataStream() { final String dataStreamName = "my-data-stream"; @@ -47,7 +55,13 @@ public void testDeleteDataStream() { ClusterState cs = DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStreamName, 2)), otherIndices); DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName }); - ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(getMetadataDeleteIndexService(), iner, cs, req); + ClusterState newState = DeleteDataStreamTransportAction.removeDataStream( + getMetadataDeleteIndexService(), + iner, + cs, + req, + systemDataStreamValidator + ); assertThat(newState.metadata().dataStreams().size(), equalTo(0)); assertThat(newState.metadata().indices().size(), equalTo(otherIndices.size())); for (String indexName : otherIndices) { @@ -68,7 +82,13 @@ public void testDeleteMultipleDataStreams() { ); DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { "ba*", "eggplant" }); - ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(getMetadataDeleteIndexService(), iner, cs, req); + ClusterState newState = DeleteDataStreamTransportAction.removeDataStream( + getMetadataDeleteIndexService(), + iner, + cs, + req, + systemDataStreamValidator + ); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); DataStream remainingDataStream = newState.metadata().dataStreams().get(dataStreamNames[0]); assertNotNull(remainingDataStream); @@ -95,7 +115,13 @@ public void testDeleteSnapshottingDataStream() { DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName }); SnapshotInProgressException e = expectThrows( SnapshotInProgressException.class, - () -> DeleteDataStreamTransportAction.removeDataStream(getMetadataDeleteIndexService(), iner, snapshotCs, req) + () -> DeleteDataStreamTransportAction.removeDataStream( + getMetadataDeleteIndexService(), + iner, + snapshotCs, + req, + systemDataStreamValidator + ) ); assertThat( @@ -144,12 +170,19 @@ public void testDeleteNonexistentDataStream() { getMetadataDeleteIndexService(), iner, cs, - new DeleteDataStreamAction.Request(new String[] { dataStreamName }) + new DeleteDataStreamAction.Request(new String[] { dataStreamName }), + systemDataStreamValidator ) ); DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName + "*" }); - ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(getMetadataDeleteIndexService(), iner, cs, req); + ClusterState newState = DeleteDataStreamTransportAction.removeDataStream( + getMetadataDeleteIndexService(), + iner, + cs, + req, + systemDataStreamValidator + ); assertThat(newState, sameInstance(cs)); assertThat(newState.metadata().dataStreams().size(), equalTo(cs.metadata().dataStreams().size())); assertThat( From dd4e9183a619a4ec3b6d43343003a5f82943f982 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 14 Apr 2021 12:39:59 -0600 Subject: [PATCH 05/22] test fixes --- .../change-mappings-and-settings.asciidoc | 3 ++- docs/reference/indices/get-data-stream.asciidoc | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index 7f596be6edfb6..0136b9d224be8 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -572,7 +572,8 @@ stream's oldest backing index. "generation": 2, "status": "GREEN", "template": "my-data-stream-template", - "hidden": false + "hidden": false, + "system": false } ] } diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index 3038c96ed50e7..2e71396e1838e 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -203,6 +203,11 @@ use the <>. `hidden`:: (Boolean) If `true`, the data stream is <>. + +`system`:: +(Boolean) +If `true`, the data stream is created and managed by an Elastic stack component +and cannot be modified through normal user interaction. ==== [[get-data-stream-api-example]] @@ -241,7 +246,8 @@ The API returns the following response: "status": "GREEN", "template": "my-index-template", "ilm_policy": "my-lifecycle-policy", - "hidden": false + "hidden": false, + "system": false }, { "name": "my-data-stream-two", @@ -261,7 +267,8 @@ The API returns the following response: "status": "YELLOW", "template": "my-index-template", "ilm_policy": "my-lifecycle-policy", - "hidden": false + "hidden": false, + "system": false } ] } From 694e31444f6912ccd2db9e711fd1a7b983a69969 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 14 Apr 2021 20:54:32 -0600 Subject: [PATCH 06/22] add ILM policy installation --- .../fleet-actions-results-ilm-policy.json | 23 +++++++ .../org/elasticsearch/xpack/fleet/Fleet.java | 34 ++++++++++ .../xpack/fleet/FleetTemplateRegistry.java | 66 +++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 x-pack/plugin/core/src/main/resources/fleet-actions-results-ilm-policy.json create mode 100644 x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java diff --git a/x-pack/plugin/core/src/main/resources/fleet-actions-results-ilm-policy.json b/x-pack/plugin/core/src/main/resources/fleet-actions-results-ilm-policy.json new file mode 100644 index 0000000000000..a65eb022fc455 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/fleet-actions-results-ilm-policy.json @@ -0,0 +1,23 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "300gb", + "max_age": "30d" + } + } + }, + "delete": { + "min_age": "90d", + "actions": { + "delete": { + "delete_searchable_snapshot": true + } + } + } + } + } +} diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 05c302f46706f..d238cc7ff2030 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -11,9 +11,12 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; @@ -22,13 +25,19 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor.Type; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.core.template.TemplateUtils; import org.elasticsearch.xpack.fleet.action.GetGlobalCheckpointsAction; import org.elasticsearch.xpack.fleet.action.GetGlobalCheckpointsShardAction; @@ -57,6 +66,31 @@ public class Fleet extends Plugin implements SystemIndexPlugin { private static final String MAPPING_VERSION_VARIABLE = "fleet.version"; private static final List ALLOWED_PRODUCTS = List.of("kibana", "fleet"); + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver expressionResolver, + Supplier repositoriesServiceSupplier + ) { + FleetTemplateRegistry registry = new FleetTemplateRegistry( + environment.settings(), + clusterService, + threadPool, + client, + xContentRegistry + ); + registry.initialize(); + return List.of(); + } + @Override public Collection getSystemIndexDescriptors(Settings settings) { return List.of( diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java new file mode 100644 index 0000000000000..37cc5fb4ee304 --- /dev/null +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.xpack.fleet; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; + +import java.util.List; + +public class FleetTemplateRegistry extends IndexTemplateRegistry { + + public static final LifecyclePolicyConfig FLEET_ACTIONS_RESULTS_POLICY = new LifecyclePolicyConfig( + ".fleet-actions-results-ilm-policy", + "/fleet-actions-results-ilm-policy.json" + ); + + public FleetTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + } + + @Override + protected String getOrigin() { + return ClientHelper.FLEET_ORIGIN; + } + + @Override + protected List getPolicyConfigs() { + return List.of(FLEET_ACTIONS_RESULTS_POLICY); + } +} From 45a3aeb9db44880209f5fc9d5e4a0b41fdeb6eb8 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 14 Apr 2021 21:00:59 -0600 Subject: [PATCH 07/22] bump value --- .../plugin/core/src/main/resources/fleet-actions-results.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/resources/fleet-actions-results.json b/x-pack/plugin/core/src/main/resources/fleet-actions-results.json index eb284e788b70c..e4a4acce782b3 100644 --- a/x-pack/plugin/core/src/main/resources/fleet-actions-results.json +++ b/x-pack/plugin/core/src/main/resources/fleet-actions-results.json @@ -32,7 +32,7 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 1024 } } }, From 00e2a13bc4fdd451736e4624fd630d4cac996432 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 15 Apr 2021 09:42:14 -0600 Subject: [PATCH 08/22] fix ilm policy --- .../fleet-actions-results-ilm-policy.json | 30 +++++++++---------- .../xpack/fleet/FleetSystemIndicesIT.java | 21 ++++++++++++- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/fleet-actions-results-ilm-policy.json b/x-pack/plugin/core/src/main/resources/fleet-actions-results-ilm-policy.json index a65eb022fc455..c0af3369413d5 100644 --- a/x-pack/plugin/core/src/main/resources/fleet-actions-results-ilm-policy.json +++ b/x-pack/plugin/core/src/main/resources/fleet-actions-results-ilm-policy.json @@ -1,21 +1,19 @@ { - "policy": { - "phases": { - "hot": { - "min_age": "0ms", - "actions": { - "rollover": { - "max_size": "300gb", - "max_age": "30d" - } + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "300gb", + "max_age": "30d" } - }, - "delete": { - "min_age": "90d", - "actions": { - "delete": { - "delete_searchable_snapshot": true - } + } + }, + "delete": { + "min_age": "90d", + "actions": { + "delete": { + "delete_searchable_snapshot": true } } } diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index 89ddc73b96135..8b6f8c9ddb8a3 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -31,11 +31,15 @@ import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.test.rest.ESRestTestCase; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import java.util.Map; + import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class FleetSystemIndicesIT extends ESRestTestCase { @@ -178,4 +182,19 @@ public void testCreationOfFleetActionsResults() throws Exception { Response response = client().performRequest(request); assertEquals(201, response.getStatusLine().getStatusCode()); } + + @SuppressWarnings("unchecked") + public void verifyILMPolicyExists() throws Exception { + assertBusy(() -> { + Request request = new Request("GET", "_ilm/policy/.fleet-actions-results-ilm-policy"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + final String responseJson = EntityUtils.toString(response.getEntity()); + Map responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), responseJson, false); + assertNotNull(responseMap.get(".fleet-actions-results-ilm-policy")); + Map policyMap = (Map) responseMap.get(".fleet-actions-results-ilm-policy"); + assertNotNull(policyMap); + assertThat(policyMap.size(), equalTo(2)); + }); + } } From da805a37dfd2440645be14300750f3bed1bc39c7 Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 15 Apr 2021 10:52:19 -0600 Subject: [PATCH 09/22] fix bug in getting datastream action and creating --- .../metadata/MetadataCreateDataStreamService.java | 2 +- .../action/GetDataStreamsTransportAction.java | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 64c82cbfee5d1..93ab9991f560a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -224,7 +224,7 @@ private static ClusterState createDataStream(MetadataCreateIndexService metadata dsBackingIndices.add(writeIndex.getIndex()); boolean hidden = isSystem ? false : template.getDataStreamTemplate().isHidden(); DataStream newDataStream = new DataStream(dataStreamName, timestampField, dsBackingIndices, 1L, - template.metadata() != null ? Map.copyOf(template.metadata()) : null, hidden, isSystem); + template.metadata() != null ? Map.copyOf(template.metadata()) : null, hidden, false, isSystem); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); logger.info("adding data stream [{}] with write index [{}] and backing indices [{}]", dataStreamName, writeIndex.getIndex().getName(), diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportAction.java index 10fb726810807..61fe39539f611 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportAction.java @@ -23,6 +23,8 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -39,6 +41,7 @@ public class GetDataStreamsTransportAction extends TransportMasterNodeReadAction GetDataStreamAction.Response> { private static final Logger LOGGER = LogManager.getLogger(GetDataStreamsTransportAction.class); + private final SystemIndices systemIndices; @Inject public GetDataStreamsTransportAction( @@ -46,7 +49,8 @@ public GetDataStreamsTransportAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + SystemIndices systemIndices ) { super( GetDataStreamAction.NAME, @@ -59,6 +63,7 @@ public GetDataStreamsTransportAction( GetDataStreamAction.Response::new, ThreadPool.Names.SAME ); + this.systemIndices = systemIndices; } @Override @@ -71,7 +76,13 @@ protected void masterOperation( List dataStreams = getDataStreams(state, indexNameExpressionResolver, request); List dataStreamInfos = new ArrayList<>(dataStreams.size()); for (DataStream dataStream : dataStreams) { - String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false); + final String indexTemplate; + if (dataStream.isSystem()) { + SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName()); + indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null; + } else { + indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false); + } String ilmPolicyName = null; if (indexTemplate != null) { Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); From 2b1b78ecf81b5bf4ba0ed5776845cbd8e8085c4e Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 15 Apr 2021 14:03:58 -0600 Subject: [PATCH 10/22] code docs --- .../admin/indices/create/AutoCreateAction.java | 3 ++- .../metadata/MetadataCreateIndexService.java | 1 + .../org/elasticsearch/indices/SystemIndices.java | 14 +++++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index f55308515813b..e358327ee6a7e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -154,7 +154,8 @@ public ClusterState execute(ClusterState currentState) throws Exception { } } - final SystemIndexDescriptor mainDescriptor = systemIndices.findMatchingDescriptor(indexName); + final SystemIndexDescriptor mainDescriptor = + isSystemIndex ? systemIndices.findMatchingDescriptor(indexName) : null; final boolean isManagedSystemIndex = mainDescriptor != null && mainDescriptor.isAutomaticallyManaged(); final CreateIndexClusterStateUpdateRequest updateRequest; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index af1dc81d1c7e5..14b273d2928b1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -341,6 +341,7 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd // The backing index may have a different name or prefix than the data stream name. final String name = request.dataStreamName() != null ? request.dataStreamName() : request.index(); + // The index being created is for a system data stream, so the backing index will also be a system index if (request.systemDataStreamDescriptor() != null) { return applyCreateIndexRequestForSystemDataStream(currentState, request, silent, metadataTransformer); } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 67205ed4a6f13..8d6f43af942c5 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -137,6 +137,10 @@ private static Map getProductToSystemIndicesMap(M new CharacterRunAutomaton(MinimizationOperations.minimize(entry.getValue(), Integer.MAX_VALUE)))); } + /** + * Checks whether the given name matches a reserved name or pattern that is intended for use by a system component. The name + * is checked against index names, aliases, data stream names, and the names of indices that back a system data stream. + */ public boolean isSystemName(String name) { return isSystemIndex(name) || isSystemDataStream(name) || isSystemIndexBackingDataStream(name); } @@ -151,7 +155,8 @@ public boolean isSystemIndex(Index index) { } /** - * Determines whether a given index is a system index by comparing its name to the collection of loaded {@link SystemIndexDescriptor}s + * Determines whether a given index is a system index by comparing its name to the collection of loaded {@link SystemIndexDescriptor}s. + * This will also match alias names that belong to system indices. * @param indexName the index name to check against loaded {@link SystemIndexDescriptor}s * @return true if the index name matches a pattern from a {@link SystemIndexDescriptor} */ @@ -159,10 +164,17 @@ public boolean isSystemIndex(String indexName) { return systemIndexAutomaton.run(indexName); } + /** + * Determines whether the provided name matches that of a system data stream that has been defined by a + * {@link SystemDataStreamDescriptor} + */ public boolean isSystemDataStream(String name) { return systemDataStreamAutomaton.test(name); } + /** + * Determines whether the provided name matches that of an index that backs a system data stream. + */ public boolean isSystemIndexBackingDataStream(String name) { return systemDataStreamIndicesAutomaton.run(name); } From 1b1a720f55191a9bf27754c40dd4eb469a2fe22b Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 15 Apr 2021 15:01:26 -0600 Subject: [PATCH 11/22] better validation for overlapping names --- .../indices/SystemIndexDescriptor.java | 7 ++- .../elasticsearch/indices/SystemIndices.java | 50 +++++++++++++++---- .../java/org/elasticsearch/node/Node.java | 12 ++--- .../org/elasticsearch/xpack/fleet/Fleet.java | 2 +- .../elasticsearch/xpack/fleet/FleetTests.java | 10 ++++ 5 files changed, 60 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index d34532bf92b42..a5912ead4b240 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -249,14 +249,17 @@ public SystemIndexDescriptor(String indexPattern, String description, Type type, this.indexPattern = indexPattern; this.primaryIndex = primaryIndex; + this.aliasName = aliasName; final Automaton automaton = buildAutomaton(indexPattern, aliasName); this.indexPatternAutomaton = new CharacterRunAutomaton(automaton); + if (primaryIndex != null && indexPatternAutomaton.run(primaryIndex) == false) { + throw new IllegalArgumentException("primary index does not match the index pattern!"); + } this.description = description; this.mappings = mappings; this.settings = settings; - this.aliasName = aliasName; this.indexFormat = indexFormat; this.versionMetaKey = versionMetaKey; this.origin = origin; @@ -384,7 +387,7 @@ public List getAllowedElasticProductOrigins() { public Version getMappingVersion() { if (type.isManaged() == false) { - throw new IllegalStateException(toString() + " is not managed so there are no mappings or version"); + throw new IllegalStateException(this + " is not managed so there are no mappings or version"); } return mappingVersion; } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 8d6f43af942c5..64aeb46185174 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -26,8 +26,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.Index; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.snapshots.SnapshotsService; import java.util.Collection; @@ -384,24 +386,31 @@ public enum SystemIndexAccessLevel { * Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed * descriptors overlap with any of the other patterns. If any do, throws an exception. * - * @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide. + * @param sourceToFeature A map of source (plugin) names to the SystemIndexDescriptors they provide. * @throws IllegalStateException Thrown if any of the index patterns overlaps with another. */ - static void checkForOverlappingPatterns(Map sourceToDescriptors) { - List> sourceDescriptorPair = sourceToDescriptors.entrySet().stream() + static void checkForOverlappingPatterns(Map sourceToFeature) { + List> sourceDescriptorPair = sourceToFeature.entrySet().stream() .flatMap(entry -> entry.getValue().getIndexDescriptors().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message .collect(Collectors.toUnmodifiableList()); + List> sourceDataStreamDescriptorPair = sourceToFeature.entrySet().stream() + .filter(entry -> entry.getValue().getDataStreamDescriptors().isEmpty() == false) + .flatMap(entry -> + entry.getValue().getDataStreamDescriptors().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) + .sorted( + Comparator.comparing(d -> d.v1() + ":" + d.v2().getDataStreamName())) // Consistent ordering -> consistent error message + .collect(Collectors.toUnmodifiableList()); // This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the // automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states // per pattern should be low as well. If these assumptions change, this might need to be reworked. sourceDescriptorPair.forEach(descriptorToCheck -> { List> descriptorsMatchingThisPattern = sourceDescriptorPair.stream() - .filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked - .filter(d -> overlaps(descriptorToCheck.v2(), d.v2())) - .collect(Collectors.toUnmodifiableList()); + .filter(d -> overlaps(descriptorToCheck.v2(), d.v2()) || + (d.v2().getAliasName() != null && descriptorToCheck.v2().matchesIndexPattern(d.v2().getAliasName()))) + .collect(toUnmodifiableList()); if (descriptorsMatchingThisPattern.isEmpty() == false) { throw new IllegalStateException("a system index descriptor [" + descriptorToCheck.v2() + "] from [" + descriptorToCheck.v1() + "] overlaps with other system index descriptors: [" + @@ -409,13 +418,28 @@ static void checkForOverlappingPatterns(Map sourceToDescriptors .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]") .collect(Collectors.joining(", "))); } + + List> dataStreamsMatching = sourceDataStreamDescriptorPair.stream() + .filter(dsTuple -> descriptorToCheck.v2().matchesIndexPattern(dsTuple.v2().getDataStreamName()) || + overlaps(descriptorToCheck.v2().getIndexPattern(), dsTuple.v2().getBackingIndexPattern())) + .collect(toUnmodifiableList()); + if (dataStreamsMatching.isEmpty() == false) { + throw new IllegalStateException("a system index descriptor [" + descriptorToCheck.v2() + "] from [" + + descriptorToCheck.v1() + "] overlaps with one or more data stream descriptors: [" + + dataStreamsMatching.stream() + .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]") + .collect(Collectors.joining(", "))); + } }); - // TODO this needs to be re-worked to account for alias, primaryIndex, and datastreams } private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) { - Automaton a1Automaton = SystemIndexDescriptor.buildAutomaton(a1.getIndexPattern(), null); - Automaton a2Automaton = SystemIndexDescriptor.buildAutomaton(a2.getIndexPattern(), null); + return overlaps(a1.getIndexPattern(), a2.getIndexPattern()); + } + + private static boolean overlaps(String pattern1, String pattern2) { + Automaton a1Automaton = SystemIndexDescriptor.buildAutomaton(pattern1, null); + Automaton a2Automaton = SystemIndexDescriptor.buildAutomaton(pattern2, null); return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false; } @@ -557,4 +581,12 @@ public void onFailure(Exception e) { }); } } + + public static Feature pluginToFeature(SystemIndexPlugin plugin, Settings settings) { + return new Feature(plugin.getFeatureDescription(), + plugin.getSystemIndexDescriptors(settings), + plugin.getSystemDataStreamDescriptors(), + plugin.getAssociatedIndexPatterns(), + plugin::cleanUpFeature); + } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 7269fcb78004a..5faf42d6eb640 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -420,15 +420,9 @@ protected Node(final Environment initialEnvironment, .stream() .peek(plugin -> SystemIndices.validateFeatureName(plugin.getFeatureName(), plugin.getClass().getCanonicalName())) .collect(Collectors.toUnmodifiableMap( - plugin -> plugin.getFeatureName(), - plugin -> new SystemIndices.Feature( - plugin.getFeatureDescription(), - plugin.getSystemIndexDescriptors(settings), - plugin.getSystemDataStreamDescriptors(), - plugin.getAssociatedIndexPatterns(), - plugin::cleanUpFeature - )) - ); + SystemIndexPlugin::getFeatureName, + plugin -> SystemIndices.pluginToFeature(plugin, settings) + )); final SystemIndices systemIndices = new SystemIndices(featuresMap); ModulesBuilder modules = new ModulesBuilder(); diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index d238cc7ff2030..6153017f2133c 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -131,7 +131,7 @@ private SystemIndexDescriptor fleetActionsSystemIndexDescriptor() { .setMappings(request.mappings()) .setSettings(request.settings()) .setPrimaryIndex(".fleet-actions-" + CURRENT_INDEX_VERSION) - .setIndexPattern(".fleet-actions*") + .setIndexPattern(".fleet-actions~(-results*)") .setAliasName(".fleet-actions") .setDescription("Fleet agents") .build(); diff --git a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java index 3bbacb1860d34..db6f7a64d973d 100644 --- a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java +++ b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java @@ -9,9 +9,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.indices.SystemIndices.Feature; import org.elasticsearch.test.ESTestCase; import java.util.Collection; +import java.util.Map; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -47,4 +50,11 @@ public void testFleetIndexNames() { assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-actions-results"))); } + + public void testFleetFeature() { + Fleet module = new Fleet(); + Feature fleet = SystemIndices.pluginToFeature(module, Settings.EMPTY); + SystemIndices systemIndices = new SystemIndices(Map.of(module.getFeatureName(), fleet)); + assertNotNull(systemIndices); + } } From 72c7b2ee1849318c5b75d85d3e01dd46f516f80a Mon Sep 17 00:00:00 2001 From: jaymode Date: Thu, 15 Apr 2021 20:03:49 -0600 Subject: [PATCH 12/22] fix test --- .../test/java/org/elasticsearch/xpack/fleet/FleetTests.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java index db6f7a64d973d..aec851d8fd947 100644 --- a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java +++ b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java @@ -32,7 +32,7 @@ public void testFleetIndexNames() { ".fleet-servers*", ".fleet-policies-[0-9]+*", ".fleet-agents*", - ".fleet-actions*", + ".fleet-actions~(-results*)", ".fleet-policies-leader*", ".fleet-enrollment-api-keys*", ".fleet-artifacts*" @@ -47,8 +47,7 @@ public void testFleetIndexNames() { assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-agents"))); assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-actions"))); - assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-actions-results"))); - + assertFalse(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-actions-results"))); } public void testFleetFeature() { From 116ca63bba665ac52adba0499136262b787ceb33 Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 16 Apr 2021 10:37:44 -0600 Subject: [PATCH 13/22] javadocs on SystemDataStreamDescriptor --- .../indices/SystemDataStreamDescriptor.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java index 66fc557496667..571c3fbe61eea 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java @@ -33,7 +33,12 @@ import java.util.List; import java.util.Map; +import java.util.Objects; +/** + * Describes a {@link DataStream} that is reserved for use by a system component. The data stream will be managed by the system and also + * protected by the system against user modification so that system features are not broken by inadvertent user operations. + */ public class SystemDataStreamDescriptor { private final String dataStreamName; @@ -47,12 +52,16 @@ public SystemDataStreamDescriptor(String dataStreamName, String description, Typ ComposableIndexTemplate composableIndexTemplate, Map componentTemplates, List allowedElasticProductOrigins) { // TODO validation and javadocs - this.dataStreamName = dataStreamName; - this.description = description; - this.type = type; - this.composableIndexTemplate = composableIndexTemplate; - this.componentTemplates = Map.copyOf(componentTemplates); - this.allowedElasticProductOrigins = allowedElasticProductOrigins; + this.dataStreamName = Objects.requireNonNull(dataStreamName, "dataStreamName must be specified"); + this.description = Objects.requireNonNull(description, "description must be specified"); + this.type = Objects.requireNonNull(type, "type must be specified"); + this.composableIndexTemplate = Objects.requireNonNull(composableIndexTemplate, "composableIndexTemplate must be provided"); + this.componentTemplates = componentTemplates == null ? Map.of() : Map.copyOf(componentTemplates); + this.allowedElasticProductOrigins = + Objects.requireNonNull(allowedElasticProductOrigins, "allowedElasticProductOrigins must not be null"); + if (type == Type.EXTERNAL && allowedElasticProductOrigins.isEmpty()) { + throw new IllegalArgumentException("External system data stream without allowed products is not a valid combination"); + } } public String getDataStreamName() { From 5ead66d78633ad73dbd5e195c6cf0c31586bf14e Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 16 Apr 2021 11:55:31 -0600 Subject: [PATCH 14/22] create system datastream unit test --- .../MetadataCreateDataStreamService.java | 24 ++---- .../MetadataMigrateToDataStreamService.java | 2 +- .../elasticsearch/indices/SystemIndices.java | 17 ++++- .../MetadataCreateDataStreamServiceTests.java | 75 ++++++++++++++++--- 4 files changed, 89 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 93ab9991f560a..7a1cafc9d7174 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectPath; import org.elasticsearch.index.Index; @@ -49,7 +48,6 @@ public class MetadataCreateDataStreamService { private final ClusterService clusterService; private final ActiveShardsObserver activeShardsObserver; private final MetadataCreateIndexService metadataCreateIndexService; - private final ThreadContext threadContext; public MetadataCreateDataStreamService(ThreadPool threadPool, ClusterService clusterService, @@ -57,7 +55,6 @@ public MetadataCreateDataStreamService(ThreadPool threadPool, this.clusterService = clusterService; this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); this.metadataCreateIndexService = metadataCreateIndexService; - this.threadContext = threadPool.getThreadContext(); } public void createDataStream(CreateDataStreamClusterStateUpdateRequest request, @@ -84,7 +81,7 @@ public void createDataStream(CreateDataStreamClusterStateUpdateRequest request, new AckedClusterStateUpdateTask(Priority.HIGH, request, listener) { @Override public ClusterState execute(ClusterState currentState) throws Exception { - ClusterState clusterState = createDataStream(metadataCreateIndexService, currentState, request, threadContext); + ClusterState clusterState = createDataStream(metadataCreateIndexService, currentState, request); firstBackingIndexRef.set(clusterState.metadata().dataStreams().get(request.name).getIndices().get(0).getName()); return clusterState; } @@ -92,7 +89,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { } public ClusterState createDataStream(CreateDataStreamClusterStateUpdateRequest request, ClusterState current) throws Exception { - return createDataStream(metadataCreateIndexService, current, request, threadContext); + return createDataStream(metadataCreateIndexService, current, request); } public static final class CreateDataStreamClusterStateUpdateRequest @@ -128,17 +125,14 @@ public SystemDataStreamDescriptor getSystemDataStreamDescriptor() { static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, ClusterState currentState, - CreateDataStreamClusterStateUpdateRequest request, - ThreadContext threadContext) throws Exception { + CreateDataStreamClusterStateUpdateRequest request) throws Exception { return createDataStream( metadataCreateIndexService, currentState, request.name, List.of(), null, - request.getSystemDataStreamDescriptor(), - threadContext - ); + request.getSystemDataStreamDescriptor()); } /** @@ -155,20 +149,18 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn ClusterState currentState, String dataStreamName, List backingIndices, - IndexMetadata writeIndex, - ThreadContext threadContext) throws Exception { + IndexMetadata writeIndex) throws Exception { assert metadataCreateIndexService.getSystemIndices().isSystemDataStream(dataStreamName) == false : "dataStream [" + dataStreamName + "] is system but no system descriptor was provided!"; - return createDataStream(metadataCreateIndexService, currentState, dataStreamName, backingIndices, writeIndex, null, threadContext); + return createDataStream(metadataCreateIndexService, currentState, dataStreamName, backingIndices, writeIndex, null); } - private static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, + static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, ClusterState currentState, String dataStreamName, List backingIndices, IndexMetadata writeIndex, - SystemDataStreamDescriptor systemDataStreamDescriptor, - ThreadContext threadContext) throws Exception + SystemDataStreamDescriptor systemDataStreamDescriptor) throws Exception { Objects.requireNonNull(metadataCreateIndexService); Objects.requireNonNull(currentState); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 190a04bf16e63..8bba3cd8e6633 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -127,7 +127,7 @@ static ClusterState migrateToDataStream(ClusterState currentState, logger.info("submitting request to migrate alias [{}] to a data stream", request.aliasName); return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, currentState, request.aliasName, - backingIndices, writeIndex, threadContext); + backingIndices, writeIndex); } // package-visible for testing diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 64aeb46185174..343ec2f45bf4f 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -487,7 +487,8 @@ public static class Feature { /** * Construct a Feature with a custom cleanup function * @param description Description of the feature - * @param indexDescriptors Patterns describing system indices for this feature + * @param indexDescriptors Collection of objects describing system indices for this feature + * @param dataStreamDescriptors Collection of objects describing system data streams for this feature * @param associatedIndexPatterns Patterns describing associated indices * @param cleanUpFunction A function that will clean up the feature's state */ @@ -516,6 +517,20 @@ public Feature(String name, String description, Collection indexDescriptors, + Collection dataStreamDescriptors) { + this(description, indexDescriptors, dataStreamDescriptors, Collections.emptyList(), + (clusterService, client, listener) -> + cleanUpFeature(indexDescriptors, Collections.emptyList(), name, clusterService, client, listener) + ); + } public String getDescription() { return description; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 7bee3aa8d7515..bbb90028b01d6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -12,11 +12,14 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.indices.EmptySystemIndices; +import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.indices.SystemDataStreamDescriptor.Type; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.indices.SystemIndices.Feature; import org.elasticsearch.test.ESTestCase; import java.util.List; @@ -27,7 +30,9 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateMapping; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Mockito.mock; @@ -48,12 +53,37 @@ public void testCreateDataStream() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); ClusterState newState = - MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY)); + MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false)); assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue()); assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"), equalTo("true")); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).isSystem(), is(false)); + } + + public void testCreateSystemDataStream() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = ".system-data-stream"; + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metadata(Metadata.builder().build()) + .build(); + CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest( + dataStreamName, systemDataStreamDescriptor(), TimeValue.ZERO, TimeValue.ZERO); + ClusterState newState = + MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); + assertThat(newState.metadata().dataStreams().size(), equalTo(1)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(true)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false)); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue()); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"), + nullValue()); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).isSystem(), is(true)); } public void testCreateDuplicateDataStream() throws Exception { @@ -68,7 +98,7 @@ public void testCreateDuplicateDataStream() throws Exception { new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] already exists")); } @@ -79,7 +109,7 @@ public void testCreateDataStreamWithInvalidName() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), containsString("must not contain the following characters")); } @@ -90,7 +120,7 @@ public void testCreateDataStreamWithUppercaseCharacters() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must be lowercase")); } @@ -101,7 +131,7 @@ public void testCreateDataStreamStartingWithPeriod() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.ds-'")); } @@ -113,7 +143,7 @@ public void testCreateDataStreamNoTemplate() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); Exception e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), equalTo("no matching index template found for data stream [my-data-stream]")); } @@ -128,7 +158,7 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); Exception e = expectThrows(IllegalArgumentException.class, - () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY))); + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); assertThat(e.getMessage(), equalTo("matching index template [template] for data stream [my-data-stream] has no data stream template")); } @@ -144,12 +174,12 @@ public static ClusterState createDataStream(final String dataStreamName) throws .build(); MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest req = new MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); - return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req, new ThreadContext(Settings.EMPTY)); + return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); } private static MetadataCreateIndexService getMetadataCreateIndexService() throws Exception { MetadataCreateIndexService s = mock(MetadataCreateIndexService.class); - when(s.getSystemIndices()).thenReturn(EmptySystemIndices.INSTANCE); + when(s.getSystemIndices()).thenReturn(getSystemIndices()); when(s.applyCreateIndexRequest(any(ClusterState.class), any(CreateIndexClusterStateUpdateRequest.class), anyBoolean())) .thenAnswer(mockInvocation -> { ClusterState currentState = (ClusterState) mockInvocation.getArguments()[0]; @@ -162,6 +192,7 @@ private static MetadataCreateIndexService getMetadataCreateIndexService() throws .put(request.settings()) .build()) .putMapping(generateMapping("@timestamp")) + .system(getSystemIndices().isSystemName(request.index())) .numberOfShards(1) .numberOfReplicas(1) .build(), false); @@ -170,4 +201,26 @@ private static MetadataCreateIndexService getMetadataCreateIndexService() throws return s; } + + private static SystemIndices getSystemIndices() { + Map map = Map.of("system", new Feature( + "systemFeature", + "system feature description", + List.of(), + List.of(systemDataStreamDescriptor()) + )); + + return new SystemIndices(map); + } + + private static SystemDataStreamDescriptor systemDataStreamDescriptor() { + return new SystemDataStreamDescriptor( + ".system-data-stream", + "test system datastream", + Type.EXTERNAL, + new ComposableIndexTemplate(List.of(".system-data-stream"), null, null, null, null, null, new DataStreamTemplate()), + Map.of(), + List.of("stack") + ); + } } From 281d04eb482d24897d3f53d02eebefc625131b00 Mon Sep 17 00:00:00 2001 From: jaymode Date: Fri, 16 Apr 2021 12:36:44 -0600 Subject: [PATCH 15/22] rollover fixes --- .../rollover/MetadataRolloverService.java | 34 +++++++++++++++---- .../metadata/MetadataCreateIndexService.java | 13 +++---- .../MetadataRolloverServiceTests.java | 12 +++---- .../TransportRolloverActionTests.java | 3 +- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 2326c218e3991..63e5a35ce01ae 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -27,6 +27,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.snapshots.SnapshotInProgressException; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.threadpool.ThreadPool; @@ -55,14 +57,17 @@ public class MetadataRolloverService { private final MetadataCreateIndexService createIndexService; private final MetadataIndexAliasesService indexAliasesService; private final IndexNameExpressionResolver indexNameExpressionResolver; + private final SystemIndices systemIndices; + @Inject public MetadataRolloverService(ThreadPool threadPool, MetadataCreateIndexService createIndexService, MetadataIndexAliasesService indexAliasesService, - IndexNameExpressionResolver indexNameExpressionResolver) { + IndexNameExpressionResolver indexNameExpressionResolver, SystemIndices systemIndices) { this.threadPool = threadPool; this.createIndexService = createIndexService; this.indexAliasesService = indexAliasesService; this.indexNameExpressionResolver = indexNameExpressionResolver; + this.systemIndices = systemIndices; } public static class RolloverResult { @@ -209,7 +214,16 @@ private RolloverResult rolloverDataStream(ClusterState currentState, IndexAbstra ); } - lookupTemplateForDataStream(dataStreamName, currentState.metadata()); + final SystemDataStreamDescriptor systemDataStreamDescriptor; + if (dataStream.isSystem() == false) { + systemDataStreamDescriptor = null; + lookupTemplateForDataStream(dataStreamName, currentState.metadata()); + } else { + systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName); + if (systemDataStreamDescriptor == null) { + throw new IllegalArgumentException("no system data stream descriptor found for data stream [" + dataStreamName + "]"); + } + } final DataStream ds = dataStream.getDataStream(); final IndexMetadata originalWriteIndex = dataStream.getWriteIndex(); @@ -219,8 +233,12 @@ private RolloverResult rolloverDataStream(ClusterState currentState, IndexAbstra return new RolloverResult(rolledDataStream.getWriteIndex().getName(), originalWriteIndex.getIndex().getName(), currentState); } - CreateIndexClusterStateUpdateRequest createIndexClusterStateRequest = - prepareDataStreamCreateIndexRequest(dataStreamName, rolledDataStream.getWriteIndex().getName(), createIndexRequest); + CreateIndexClusterStateUpdateRequest createIndexClusterStateRequest = prepareDataStreamCreateIndexRequest( + dataStreamName, + rolledDataStream.getWriteIndex().getName(), + createIndexRequest, + systemDataStreamDescriptor + ); ClusterState newState = createIndexService.applyCreateIndexRequest(currentState, createIndexClusterStateRequest, silent, (builder, indexMetadata) -> builder.put(ds.rollover(currentState.metadata(), indexMetadata.getIndexUUID()))); @@ -252,10 +270,12 @@ static String generateRolloverIndexName(String sourceIndexName, IndexNameExpress static CreateIndexClusterStateUpdateRequest prepareDataStreamCreateIndexRequest(final String dataStreamName, final String targetIndexName, - CreateIndexRequest createIndexRequest) { - Settings settings = Settings.builder().put("index.hidden", true).build(); + CreateIndexRequest createIndexRequest, + final SystemDataStreamDescriptor descriptor) { + Settings settings = descriptor != null ? Settings.EMPTY : Settings.builder().put("index.hidden", true).build(); return prepareCreateIndexRequest(targetIndexName, targetIndexName, "rollover_data_stream", createIndexRequest, settings) - .dataStreamName(dataStreamName); + .dataStreamName(dataStreamName) + .systemDataStreamDescriptor(descriptor); } static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 14b273d2928b1..b0c6ab0ebfbe5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -332,12 +332,6 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd // in which case templates don't apply, so create the index from the source metadata return applyCreateIndexRequestWithExistingMetadata(currentState, request, silent, sourceMetadata, metadataTransformer); } else { - // Hidden indices apply templates slightly differently (ignoring wildcard '*' - // templates), so we need to check to see if the request is creating a hidden index - // prior to resolving which templates it matches - final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings()) ? - IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; - // The backing index may have a different name or prefix than the data stream name. final String name = request.dataStreamName() != null ? request.dataStreamName() : request.index(); @@ -345,6 +339,13 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd if (request.systemDataStreamDescriptor() != null) { return applyCreateIndexRequestForSystemDataStream(currentState, request, silent, metadataTransformer); } + + // Hidden indices apply templates slightly differently (ignoring wildcard '*' + // templates), so we need to check to see if the request is creating a hidden index + // prior to resolving which templates it matches + final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings()) ? + IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; + // Check to see if a v2 template matched final String v2Template = MetadataIndexTemplateService.findV2Template(currentState.metadata(), name, isHiddenFromRequest == null ? false : isHiddenFromRequest); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 51498ac29321b..e663762669e21 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -335,7 +335,7 @@ public void testCreateIndexRequestForDataStream() { .build(); rolloverRequest.getCreateIndexRequest().settings(settings); final CreateIndexClusterStateUpdateRequest createIndexRequest = MetadataRolloverService.prepareDataStreamCreateIndexRequest( - dataStream.getName(), newWriteIndexName, rolloverRequest.getCreateIndexRequest()); + dataStream.getName(), newWriteIndexName, rolloverRequest.getCreateIndexRequest(), null); for (String settingKey : settings.keySet()) { assertThat(settings.get(settingKey), equalTo(createIndexRequest.settings().get(settingKey))); } @@ -503,7 +503,7 @@ public void testRolloverClusterState() throws Exception { MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry()); MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, - mockIndexNameExpressionResolver); + mockIndexNameExpressionResolver, EmptySystemIndices.INSTANCE); MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); List> metConditions = Collections.singletonList(condition); @@ -607,7 +607,7 @@ protected String contentType() { MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry()); MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, - mockIndexNameExpressionResolver); + mockIndexNameExpressionResolver, EmptySystemIndices.INSTANCE); MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); List> metConditions = Collections.singletonList(condition); @@ -709,7 +709,7 @@ clusterService, indicesService, allocationService, new AliasValidator(), shardLi MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry()); MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, - mockIndexNameExpressionResolver); + mockIndexNameExpressionResolver, EmptySystemIndices.INSTANCE); MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); List> metConditions = Collections.singletonList(condition); @@ -764,7 +764,7 @@ public void testValidation() throws Exception { IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class); when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg()); MetadataRolloverService rolloverService = new MetadataRolloverService(null, createIndexService, metadataIndexAliasesService, - mockIndexNameExpressionResolver); + mockIndexNameExpressionResolver, EmptySystemIndices.INSTANCE); String newIndexName = useDataStream == false && randomBoolean() ? "logs-index-9" : null; @@ -814,7 +814,7 @@ public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception { MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry()); MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, - mockIndexNameExpressionResolver); + mockIndexNameExpressionResolver, EmptySystemIndices.INSTANCE); MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); List> metConditions = Collections.singletonList(condition); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 69a62059ed385..7a0d688d94039 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -52,6 +52,7 @@ import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.warmer.WarmerStats; +import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; @@ -248,7 +249,7 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr when(mockCreateIndexService.applyCreateIndexRequest(any(), any(), anyBoolean())).thenReturn(stateBefore); when(mdIndexAliasesService.applyAliasActions(any(), any())).thenReturn(stateBefore); MetadataRolloverService rolloverService = new MetadataRolloverService(mockThreadPool, mockCreateIndexService, - mdIndexAliasesService, mockIndexNameExpressionResolver); + mdIndexAliasesService, mockIndexNameExpressionResolver, EmptySystemIndices.INSTANCE); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction(mockTransportService, mockClusterService, mockThreadPool, mockActionFilters, mockIndexNameExpressionResolver, rolloverService, mockClient); From d5fda7ded947938b4408d14cfdb2c31f2c4c4012 Mon Sep 17 00:00:00 2001 From: jaymode Date: Mon, 19 Apr 2021 20:41:04 -0600 Subject: [PATCH 16/22] more test and feedback --- .../client/indices/DataStream.java | 3 +- .../cluster/metadata/IndexAbstraction.java | 3 +- .../metadata/IndexNameExpressionResolver.java | 145 ++++--- .../MetadataCreateDataStreamService.java | 2 +- .../MetadataIndexTemplateService.java | 2 +- .../indices/SystemDataStreamDescriptor.java | 13 +- .../elasticsearch/indices/SystemIndices.java | 27 +- .../DateMathExpressionResolverTests.java | 5 +- .../IndexNameExpressionResolverTests.java | 6 +- .../WildcardExpressionResolverTests.java | 5 +- .../core/ilm/GenerateSnapshotNameStep.java | 3 +- x-pack/plugin/data-streams/build.gradle | 4 +- .../datastreams/SystemDataStreamIT.java | 400 ++++++++++++++++++ .../DeleteDataStreamTransportAction.java | 25 +- .../action/GetDataStreamsTransportAction.java | 30 +- .../DeleteDataStreamTransportActionTests.java | 43 +- .../org/elasticsearch/xpack/fleet/Fleet.java | 32 ++ 17 files changed, 613 insertions(+), 135 deletions(-) create mode 100644 x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java index 691949e5eeb2d..db27600a5cc9c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java @@ -114,8 +114,7 @@ public boolean isSystem() { String indexTemplate = (String) args[5]; String ilmPolicy = (String) args[6]; Map metadata = (Map) args[7]; - Boolean hidden = (Boolean) args[8]; - hidden = hidden != null && hidden; + boolean hidden = args[8] != null && (boolean) args[8]; boolean system = args[9] != null && (boolean) args[9]; return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata, hidden, system); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index 845068a007c4d..f285191854a56 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -317,8 +317,7 @@ public boolean isHidden() { @Override public boolean isSystem() { - // No such thing as system data streams (yet) - return false; + return dataStream.isSystem(); } public org.elasticsearch.cluster.metadata.DataStream getDataStream() { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 9c54627f4e1fc..a73fa17bf44ec 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexAbstraction.Type; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -77,7 +78,7 @@ public IndexNameExpressionResolver(ThreadContext threadContext, SystemIndices sy */ public String[] concreteIndexNames(ClusterState state, IndicesRequest request) { Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams(), - getSystemIndexAccessLevel()); + getSystemIndexAccessPredicate()); return concreteIndexNames(context, request.indices()); } @@ -86,7 +87,7 @@ public String[] concreteIndexNames(ClusterState state, IndicesRequest request) { */ public String[] concreteIndexNamesWithSystemIndexAccess(ClusterState state, IndicesRequest request) { Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams(), - SystemIndexAccessLevel.ALL); + name -> true); return concreteIndexNames(context, request.indices()); } @@ -96,7 +97,7 @@ public String[] concreteIndexNamesWithSystemIndexAccess(ClusterState state, Indi */ public Index[] concreteIndices(ClusterState state, IndicesRequest request) { Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams(), - getSystemIndexAccessLevel()); + getSystemIndexAccessPredicate()); return concreteIndices(context, request.indices()); } @@ -114,28 +115,27 @@ public Index[] concreteIndices(ClusterState state, IndicesRequest request) { * indices options in the context don't allow such a case. */ public String[] concreteIndexNames(ClusterState state, IndicesOptions options, String... indexExpressions) { - Context context = new Context(state, options, getSystemIndexAccessLevel()); + Context context = new Context(state, options, getSystemIndexAccessPredicate()); return concreteIndexNames(context, indexExpressions); } public String[] concreteIndexNames(ClusterState state, IndicesOptions options, boolean includeDataStreams, String... indexExpressions) { - Context context = new Context(state, options, false, false, includeDataStreams, getSystemIndexAccessLevel()); + Context context = new Context(state, options, false, false, includeDataStreams, getSystemIndexAccessPredicate()); return concreteIndexNames(context, indexExpressions); } public String[] concreteIndexNames(ClusterState state, IndicesOptions options, IndicesRequest request) { - Context context = new Context(state, options, false, false, request.includeDataStreams(), getSystemIndexAccessLevel()); + Context context = new Context(state, options, false, false, request.includeDataStreams(), getSystemIndexAccessPredicate()); return concreteIndexNames(context, request.indices()); } public String[] concreteIndexNamesWithSystemIndexAccess(ClusterState state, IndicesOptions options, String... indexExpressions) { - Context context = new Context(state, options, SystemIndexAccessLevel.ALL); + Context context = new Context(state, options, name -> true); return concreteIndexNames(context, indexExpressions); } public List dataStreamNames(ClusterState state, IndicesOptions options, String... indexExpressions) { - // Allow system index access - they'll be filtered out below as there's no such thing (yet) as system data streams - Context context = new Context(state, options, false, false, true, true, SystemIndexAccessLevel.ALL); + Context context = new Context(state, options, false, false, true, true, getSystemIndexAccessPredicate()); if (indexExpressions == null || indexExpressions.length == 0) { indexExpressions = new String[]{"*"}; } @@ -168,7 +168,7 @@ public Index[] concreteIndices(ClusterState state, IndicesOptions options, Strin public Index[] concreteIndices(ClusterState state, IndicesOptions options, boolean includeDataStreams, String... indexExpressions) { Context context = new Context(state, options, false, false, includeDataStreams, - getSystemIndexAccessLevel()); + getSystemIndexAccessPredicate()); return concreteIndices(context, indexExpressions); } @@ -186,7 +186,7 @@ public Index[] concreteIndices(ClusterState state, IndicesOptions options, boole */ public Index[] concreteIndices(ClusterState state, IndicesRequest request, long startTime) { Context context = new Context(state, request.indicesOptions(), startTime, false, false, request.includeDataStreams(), false, - getSystemIndexAccessLevel()); + getSystemIndexAccessPredicate()); return concreteIndices(context, request.indices()); } @@ -313,29 +313,38 @@ Index[] concreteIndices(Context context, String... indexExpressions) { } private void checkSystemIndexAccess(Context context, Metadata metadata, Set concreteIndices, String[] originalPatterns) { - final SystemIndexAccessLevel systemIndexAccessLevel = context.getSystemIndexAccessLevel(); - if (systemIndexAccessLevel != SystemIndexAccessLevel.ALL) { - final Predicate systemIndexAccessLevelPredicate; - if (systemIndexAccessLevel == SystemIndexAccessLevel.NONE) { - // everything should be included in the deprecation message - systemIndexAccessLevelPredicate = indexMetadata -> true; + final Predicate systemIndexAccessPredicate = context.getSystemIndexAccessPredicate().negate(); + final List matchedSystemIndexMetadata = concreteIndices.stream() + .map(metadata::index) + .filter(IndexMetadata::isSystem) + .filter(idxMetadata -> systemIndexAccessPredicate.test(idxMetadata.getIndex().getName())) + .collect(Collectors.toList()); + + if (matchedSystemIndexMetadata.isEmpty()) { + return; + } + + final List resolvedSystemIndices = new ArrayList<>(); + final Set resolvedSystemDataStreams = new HashSet<>(); + final SortedMap indicesLookup = metadata.getIndicesLookup(); + for (IndexMetadata idxMetadata : matchedSystemIndexMetadata) { + IndexAbstraction abstraction = indicesLookup.get(idxMetadata.getIndex().getName()); + if (abstraction.getParentDataStream() != null) { + resolvedSystemDataStreams.add(abstraction.getParentDataStream().getName()); } else { - // everything other than allowed should be included in the deprecation message - systemIndexAccessLevelPredicate = systemIndices.getProductSystemIndexMetadataPredicate(threadContext).negate(); - } - final List resolvedSystemIndices = concreteIndices.stream() - .map(metadata::index) - .filter(IndexMetadata::isSystem) - .filter(systemIndexAccessLevelPredicate) - .map(i -> i.getIndex().getName()) - .sorted() // reliable order for testing - .collect(Collectors.toList()); - if (resolvedSystemIndices.isEmpty() == false) { - deprecationLogger.deprecate(DeprecationCategory.API, "open_system_index_access", - "this request accesses system indices: {}, but in a future major version, direct access to system " + - "indices will be prevented by default", resolvedSystemIndices); + resolvedSystemIndices.add(idxMetadata.getIndex().getName()); } } + + if (resolvedSystemIndices.isEmpty() == false) { + Collections.sort(resolvedSystemIndices); + deprecationLogger.deprecate(DeprecationCategory.API, "open_system_index_access", + "this request accesses system indices: {}, but in a future major version, direct access to system " + + "indices will be prevented by default", resolvedSystemIndices); + } + if (resolvedSystemDataStreams.isEmpty() == false) { + throw systemIndices.dataStreamAccessException(threadContext, resolvedSystemDataStreams); + } } private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, IndexMetadata index) { @@ -422,7 +431,7 @@ public Index concreteWriteIndex(ClusterState state, IndicesOptions options, Stri options.allowAliasesToMultipleIndices(), options.forbidClosedIndices(), options.ignoreAliases(), options.ignoreThrottled()); - Context context = new Context(state, combinedOptions, false, true, includeDataStreams, getSystemIndexAccessLevel()); + Context context = new Context(state, combinedOptions, false, true, includeDataStreams, getSystemIndexAccessPredicate()); Index[] indices = concreteIndices(context, index); if (allowNoIndices && indices.length == 0) { return null; @@ -439,7 +448,7 @@ public Index concreteWriteIndex(ClusterState state, IndicesOptions options, Stri * If the data stream, index or alias contains date math then that is resolved too. */ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) { - Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true, getSystemIndexAccessLevel()); + Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true, getSystemIndexAccessPredicate()); String resolvedAliasOrIndex = DateMathExpressionResolver.resolveExpression(indexAbstraction, context); return state.metadata().getIndicesLookup().containsKey(resolvedAliasOrIndex); } @@ -450,7 +459,7 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) public String resolveDateMathExpression(String dateExpression) { // The data math expression resolver doesn't rely on cluster state or indices options, because // it just resolves the date math to an actual date. - return DateMathExpressionResolver.resolveExpression(dateExpression, new Context(null, null, getSystemIndexAccessLevel())); + return DateMathExpressionResolver.resolveExpression(dateExpression, new Context(null, null, getSystemIndexAccessPredicate())); } /** @@ -458,14 +467,14 @@ public String resolveDateMathExpression(String dateExpression) { * @return If the specified string is data math expression then this method returns the resolved expression. */ public String resolveDateMathExpression(String dateExpression, long time) { - return DateMathExpressionResolver.resolveExpression(dateExpression, new Context(null, null, time, getSystemIndexAccessLevel())); + return DateMathExpressionResolver.resolveExpression(dateExpression, new Context(null, null, time, getSystemIndexAccessPredicate())); } /** * Resolve an array of expressions to the set of indices and aliases that these expressions match. */ public Set resolveExpressions(ClusterState state, String... expressions) { - Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false, true, getSystemIndexAccessLevel()); + Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false, true, getSystemIndexAccessPredicate()); List resolvedExpressions = Arrays.asList(expressions); for (ExpressionResolver expressionResolver : expressionResolvers) { resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions); @@ -559,7 +568,7 @@ public String[] indexAliases(ClusterState state, String index, Predicate> resolveSearchRouting(ClusterState state, @Nullable String routing, String... expressions) { List resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.emptyList(); - Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true, getSystemIndexAccessLevel()); + Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true, getSystemIndexAccessPredicate()); for (ExpressionResolver expressionResolver : expressionResolvers) { resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions); } @@ -714,6 +723,20 @@ public SystemIndexAccessLevel getSystemIndexAccessLevel() { return systemIndices.getSystemIndexAccessLevel(threadContext); } + public Predicate getSystemIndexAccessPredicate() { + final SystemIndexAccessLevel systemIndexAccessLevel = getSystemIndexAccessLevel(); + final Predicate systemIndexAccessLevelPredicate; + if (systemIndexAccessLevel == SystemIndexAccessLevel.NONE) { + systemIndexAccessLevelPredicate = s -> false; + } else if (systemIndexAccessLevel == SystemIndexAccessLevel.ALL) { + systemIndexAccessLevelPredicate = s -> true; + } else { + // everything other than allowed should be included in the deprecation message + systemIndexAccessLevelPredicate = systemIndices.getProductSystemIndexNamePredicate(threadContext); + } + return systemIndexAccessLevelPredicate; + } + public static class Context { private final ClusterState state; @@ -723,30 +746,30 @@ public static class Context { private final boolean resolveToWriteIndex; private final boolean includeDataStreams; private final boolean preserveDataStreams; - private final SystemIndexAccessLevel systemIndexAccessLevel; + private final Predicate systemIndexAccessPredicate; - Context(ClusterState state, IndicesOptions options, SystemIndexAccessLevel systemIndexAccessLevel) { - this(state, options, System.currentTimeMillis(), systemIndexAccessLevel); + Context(ClusterState state, IndicesOptions options, Predicate systemIndexAccessPredicate) { + this(state, options, System.currentTimeMillis(), systemIndexAccessPredicate); } Context(ClusterState state, IndicesOptions options, boolean preserveAliases, boolean resolveToWriteIndex, - boolean includeDataStreams, SystemIndexAccessLevel systemIndexAccessLevel) { + boolean includeDataStreams, Predicate systemIndexAccessPredicate) { this(state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex, includeDataStreams, false, - systemIndexAccessLevel); + systemIndexAccessPredicate); } Context(ClusterState state, IndicesOptions options, boolean preserveAliases, boolean resolveToWriteIndex, - boolean includeDataStreams, boolean preserveDataStreams, SystemIndexAccessLevel systemIndexAccessLevel) { + boolean includeDataStreams, boolean preserveDataStreams, Predicate systemIndexAccessPredicate) { this(state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex, includeDataStreams, preserveDataStreams, - systemIndexAccessLevel); + systemIndexAccessPredicate); } - Context(ClusterState state, IndicesOptions options, long startTime, SystemIndexAccessLevel systemIndexAccessLevel) { - this(state, options, startTime, false, false, false, false, systemIndexAccessLevel); + Context(ClusterState state, IndicesOptions options, long startTime, Predicate systemIndexAccessPredicate) { + this(state, options, startTime, false, false, false, false, systemIndexAccessPredicate); } protected Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex, - boolean includeDataStreams, boolean preserveDataStreams, SystemIndexAccessLevel systemIndexAccessLevel) { + boolean includeDataStreams, boolean preserveDataStreams, Predicate systemIndexAccessPredicate) { this.state = state; this.options = options; this.startTime = startTime; @@ -754,7 +777,7 @@ protected Context(ClusterState state, IndicesOptions options, long startTime, bo this.resolveToWriteIndex = resolveToWriteIndex; this.includeDataStreams = includeDataStreams; this.preserveDataStreams = preserveDataStreams; - this.systemIndexAccessLevel = systemIndexAccessLevel; + this.systemIndexAccessPredicate = systemIndexAccessPredicate; } public ClusterState getState() { @@ -797,8 +820,8 @@ public boolean isPreserveDataStreams() { /** * Used to determine system index access is allowed in this context (e.g. for this request). */ - public SystemIndexAccessLevel getSystemIndexAccessLevel() { - return systemIndexAccessLevel; + public Predicate getSystemIndexAccessPredicate() { + return systemIndexAccessPredicate; } } @@ -830,7 +853,20 @@ public List resolve(Context context, List expressions) { } if (isEmptyOrTrivialWildcard(expressions)) { - List resolvedExpressions = resolveEmptyOrTrivialWildcard(options, metadata); + List resolvedExpressions = resolveEmptyOrTrivialWildcard(options, metadata).stream() + .filter(expression -> { + IndexAbstraction abstraction = metadata.getIndicesLookup().get(expression); + if (abstraction != null && abstraction.isSystem()) { + if (abstraction.getType() == Type.DATA_STREAM || abstraction.getParentDataStream() != null) { + return context.systemIndexAccessPredicate.test(abstraction.getName()); + } else { + return true; + } + } else { + return true; + } + }) + .collect(Collectors.toList()); if (context.includeDataStreams()) { final IndexMetadata.State excludeState = excludeState(options); final Map dataStreamsAbstractions = metadata.getIndicesLookup().entrySet() @@ -1030,6 +1066,13 @@ private static Set expand(Context context, IndexMetadata.State excludeSt String aliasOrIndexName = entry.getKey(); IndexAbstraction indexAbstraction = entry.getValue(); + if (indexAbstraction.isSystem() && + (indexAbstraction.getType() == Type.DATA_STREAM || indexAbstraction.getParentDataStream() != null)) { + if (context.systemIndexAccessPredicate.test(indexAbstraction.getName()) == false) { + continue; + } + } + if (indexAbstraction.isHidden() == false || includeHidden || implicitHiddenMatch(aliasOrIndexName, expression)) { if (context.isPreserveAliases() && indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) { expand.add(aliasOrIndexName); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 7a1cafc9d7174..e138937fc26fe 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -179,7 +179,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '" + DataStream.BACKING_INDEX_PREFIX + "'"); } -; + final boolean isSystem = systemDataStreamDescriptor != null; final ComposableIndexTemplate template = isSystem ? systemDataStreamDescriptor.getComposableIndexTemplate() : diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 619a843b7f25b..9aaab63164c8d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1074,7 +1074,7 @@ public static Settings resolveSettings(final Metadata metadata, final String tem /** * Resolve the provided v2 template and component templates into a collected {@link Settings} object */ - static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { + public static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { Objects.requireNonNull(template, "attempted to resolve settings for a null template"); Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); List componentSettings = template.composedOf().stream() diff --git a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java index 571c3fbe61eea..3cf571ca7f939 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java @@ -48,10 +48,21 @@ public class SystemDataStreamDescriptor { private final Map componentTemplates; private final List allowedElasticProductOrigins; + /** + * Creates a new descriptor for a system data descriptor + * @param dataStreamName the name of the data stream. Must not be {@code null} + * @param description a brief description of what the data stream is used for. Must not be {@code null} + * @param type the {@link Type} of the data stream which determines how the data stream can be accessed. Must not be {@code null} + * @param composableIndexTemplate the {@link ComposableIndexTemplate} that contains the mappings and settings for the data stream. + * Must not be {@code null} + * @param componentTemplates a map that contains {@link ComponentTemplate} instances corresponding to those references in the + * {@link ComposableIndexTemplate} + * @param allowedElasticProductOrigins a list of product origin values that are allowed to access this data stream if the + * type is {@link Type#EXTERNAL}. Must not be {@code null} + */ public SystemDataStreamDescriptor(String dataStreamName, String description, Type type, ComposableIndexTemplate composableIndexTemplate, Map componentTemplates, List allowedElasticProductOrigins) { - // TODO validation and javadocs this.dataStreamName = Objects.requireNonNull(dataStreamName, "dataStreamName must be specified"); this.description = Objects.requireNonNull(description, "description must be specified"); this.type = Objects.requireNonNull(type, "type must be specified"); diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 343ec2f45bf4f..80eb97ddf10f3 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -32,6 +32,7 @@ import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.snapshots.SnapshotsService; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -329,15 +330,16 @@ public SystemDataStreamDescriptor validateDataStreamAccess(String dataStreamName .flatMap(feature -> feature.getDataStreamDescriptors().stream()) .filter(descriptor -> descriptor.getDataStreamName().equals(dataStreamName)) .findFirst() - .orElseThrow(() -> new IllegalStateException("system datastream descriptor not found for [" + dataStreamName + "]")); + .orElseThrow(() -> new IllegalStateException("system data stream descriptor not found for [" + dataStreamName + "]")); if (dataStreamDescriptor.isExternal()) { final SystemIndexAccessLevel accessLevel = getSystemIndexAccessLevel(threadContext); if (accessLevel == SystemIndexAccessLevel.NONE) { - throw new IllegalArgumentException("DataStream [" + dataStreamName + "] is reserved for system use and access"); + throw dataStreamAccessException(null, dataStreamName); } else if (accessLevel == SystemIndexAccessLevel.RESTRICTED) { if (getProductSystemIndexNamePredicate(threadContext).test(dataStreamName) == false) { - throw new IllegalArgumentException("DataStream [" + dataStreamName + "] is not accessible by product [" + - threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY) + "]"); + throw dataStreamAccessException( + threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY), + dataStreamName); } else { return dataStreamDescriptor; } @@ -353,6 +355,23 @@ public SystemDataStreamDescriptor validateDataStreamAccess(String dataStreamName } } + public IllegalArgumentException dataStreamAccessException(ThreadContext threadContext, Collection names) { + return dataStreamAccessException( + threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY), + names.toArray(Strings.EMPTY_ARRAY) + ); + } + + IllegalArgumentException dataStreamAccessException(@Nullable String product, String... dataStreamNames) { + if (product == null) { + return new IllegalArgumentException("Data stream(s) " + Arrays.toString(dataStreamNames) + + " use and access is reserved for system operations"); + } else { + return new IllegalArgumentException("Data stream(s) " + Arrays.toString(dataStreamNames) + " may not be accessed by product [" + + product + "]"); + } + } + /** * Determines what level of system index access should be allowed in the current context. * diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java index 5b74e46b0f671..b594eb31fe844 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.List; -import static org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel.NONE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.joda.time.DateTimeZone.UTC; @@ -34,7 +33,7 @@ public class DateMathExpressionResolverTests extends ESTestCase { private final DateMathExpressionResolver expressionResolver = new DateMathExpressionResolver(); private final Context context = new Context( ClusterState.builder(new ClusterName("_name")).build(), IndicesOptions.strictExpand(), - NONE + name -> false ); public void testNormal() throws Exception { @@ -137,7 +136,7 @@ public void testExpression_CustomTimeZoneInIndexName() throws Exception { // rounding to today 00:00 now = DateTime.now(UTC).withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0); } - Context context = new Context(this.context.getState(), this.context.getOptions(), now.getMillis(), NONE); + Context context = new Context(this.context.getState(), this.context.getOptions(), now.getMillis(), name -> false); List results = expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{yyyy.MM.dd|" + timeZone.getID() + "}}>")); assertThat(results.size(), equalTo(1)); logger.info("timezone: [{}], now [{}], name: [{}]", timeZone, now, results.get(0)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 3b52bbafc93d7..e536affef78bb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; - import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; @@ -49,6 +48,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; @@ -57,7 +57,6 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.elasticsearch.indices.SystemIndices.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; import static org.elasticsearch.indices.SystemIndices.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; -import static org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel.NONE; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.arrayWithSize; @@ -72,6 +71,9 @@ import static org.hamcrest.Matchers.notNullValue; public class IndexNameExpressionResolverTests extends ESTestCase { + + private static final Predicate NONE = name -> false; + private IndexNameExpressionResolver indexNameExpressionResolver; private ThreadContext threadContext; private long epochMillis; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java index 5ab3104491a66..5d1bfcdb12640 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -20,15 +20,18 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.function.Predicate; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.common.util.set.Sets.newHashSet; -import static org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel.NONE; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; public class WildcardExpressionResolverTests extends ESTestCase { + + private static final Predicate NONE = name -> false; + public void testConvertWildcardsJustIndicesTests() { Metadata.Builder mdBuilder = Metadata.builder() .put(indexBuilder("testXXX")) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java index d78097475603f..95934d780ff62 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.index.Index; -import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import java.util.Collections; import java.util.List; @@ -140,7 +139,7 @@ public ResolverContext() { } public ResolverContext(long startTime) { - super(null, null, startTime, false, false, false, false, SystemIndexAccessLevel.NONE); + super(null, null, startTime, false, false, false, false, name -> false); } @Override diff --git a/x-pack/plugin/data-streams/build.gradle b/x-pack/plugin/data-streams/build.gradle index d8f7c499baa06..785e418a20053 100644 --- a/x-pack/plugin/data-streams/build.gradle +++ b/x-pack/plugin/data-streams/build.gradle @@ -11,6 +11,8 @@ archivesBaseName = 'x-pack-data-streams' dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(path: ':modules:transport-netty4') // for http in SystemDataStreamIT + testImplementation project(path: ':plugins:transport-nio') // for http in SystemDataStreamIT } -addQaCheckDependencies() \ No newline at end of file +addQaCheckDependencies() diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java new file mode 100644 index 0000000000000..8bda8f06e4527 --- /dev/null +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java @@ -0,0 +1,400 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.datastreams; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.IndicesOptions.Option; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.indices.SystemDataStreamDescriptor.Type; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.Netty4Plugin; +import org.elasticsearch.transport.nio.NioTransportPlugin; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; +import org.junit.After; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class SystemDataStreamIT extends ESIntegTestCase { + + private static String nodeHttpTypeKey; + + @BeforeClass + public static void setUpTransport() { + if (randomBoolean()) { + nodeHttpTypeKey = NioTransportPlugin.NIO_HTTP_TRANSPORT_NAME; + } else { + nodeHttpTypeKey = Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME; + } + } + + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(NioTransportPlugin.class); + plugins.add(DataStreamsPlugin.class); + plugins.add(TestSystemDataStreamPlugin.class); + plugins.add(Netty4Plugin.class); + return plugins; + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(NetworkModule.HTTP_TYPE_KEY, nodeHttpTypeKey) + .build(); + } + + @Override + protected boolean addMockHttpTransport() { + return false; + } + + @SuppressWarnings("unchecked") + public void testSystemDataStreamCRUD() throws Exception { + try (RestClient restClient = createRestClient()) { + Request putRequest = new Request("PUT", "/_data_stream/.test-data-stream"); + + // no product header + ResponseException re = expectThrows(ResponseException.class, () -> restClient.performRequest(putRequest)); + assertThat(re.getMessage(), containsString("reserved for system")); + + // wrong header + putRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "wrong").build()); + re = expectThrows(ResponseException.class, () -> restClient.performRequest(putRequest)); + assertThat(re.getMessage(), containsString("accessed by product")); + + // correct + putRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "product").build()); + Response putResponse = restClient.performRequest(putRequest); + assertThat(putResponse.getStatusLine().getStatusCode(), is(200)); + + // list - no header needed + Request listAllRequest = new Request("GET", "/_data_stream"); + Response listAllResponse = restClient.performRequest(listAllRequest); + assertThat(listAllResponse.getStatusLine().getStatusCode(), is(200)); + Map responseMap = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + EntityUtils.toString(listAllResponse.getEntity()), + false + ); + List dataStreams = (List) responseMap.get("data_streams"); + assertThat(dataStreams.size(), is(1)); + + Request listRequest = new Request("GET", "/_data_stream/.test-data-stream"); + Response listResponse = restClient.performRequest(listRequest); + assertThat(listResponse.getStatusLine().getStatusCode(), is(200)); + responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), EntityUtils.toString(listResponse.getEntity()), false); + dataStreams = (List) responseMap.get("data_streams"); + assertThat(dataStreams.size(), is(1)); + + // delete + Request deleteRequest = new Request("DELETE", "/_data_stream/.test-data-stream"); + + // no header + re = expectThrows(ResponseException.class, () -> restClient.performRequest(deleteRequest)); + assertThat(re.getMessage(), containsString("reserved for system")); + + // incorrect header + deleteRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "wrong").build()); + re = expectThrows(ResponseException.class, () -> restClient.performRequest(deleteRequest)); + assertThat(re.getMessage(), containsString("accessed by product")); + + // correct + deleteRequest.setOptions(putRequest.getOptions()); + Response deleteResponse = restClient.performRequest(deleteRequest); + assertThat(deleteResponse.getStatusLine().getStatusCode(), is(200)); + } + } + + public void testDataStreamStats() throws Exception { + try (RestClient restClient = createRestClient()) { + Request putRequest = new Request("PUT", "/_data_stream/.test-data-stream"); + putRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "product").build()); + Response putResponse = restClient.performRequest(putRequest); + assertThat(putResponse.getStatusLine().getStatusCode(), is(200)); + + Request statsRequest = new Request("GET", "/_data_stream/_stats"); + Response response = restClient.performRequest(statsRequest); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + + Map map = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + EntityUtils.toString(response.getEntity()), + false + ); + assertThat(map.get("data_stream_count"), equalTo(1)); + } + } + + @SuppressWarnings("unchecked") + public void testSystemDataStreamReadWrite() throws Exception { + try (RestClient restClient = createRestClient()) { + Request putRequest = new Request("PUT", "/_data_stream/.test-data-stream"); + putRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "product").build()); + Response putResponse = restClient.performRequest(putRequest); + assertThat(putResponse.getStatusLine().getStatusCode(), is(200)); + + // write + Request index = new Request("POST", "/.test-data-stream/_doc"); + index.setJsonEntity("{ \"@timestamp\": \"2099-03-08T11:06:07.000Z\", \"name\": \"my-name\" }"); + index.addParameter("refresh", "true"); + + // no product specified + ResponseException re = expectThrows(ResponseException.class, () -> restClient.performRequest(index)); + assertThat(re.getMessage(), containsString("reserved for system")); + + // wrong header + index.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "wrong").build()); + re = expectThrows(ResponseException.class, () -> restClient.performRequest(index)); + assertThat(re.getMessage(), containsString("accessed by product")); + + // correct + index.setOptions(putRequest.getOptions()); + Response response = restClient.performRequest(index); + assertEquals(201, response.getStatusLine().getStatusCode()); + + Map responseMap = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + EntityUtils.toString(response.getEntity()), + false + ); + String indexName = (String) responseMap.get("_index"); + String id = (String) responseMap.get("_id"); + + // get + Request get = new Request("GET", "/" + indexName + "/_doc/" + id); + + // no product specified + re = expectThrows(ResponseException.class, () -> restClient.performRequest(get)); + assertThat(re.getMessage(), containsString("reserved for system")); + + // wrong product + get.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "wrong").build()); + re = expectThrows(ResponseException.class, () -> restClient.performRequest(get)); + assertThat(re.getMessage(), containsString("accessed by product")); + + // correct + get.setOptions(putRequest.getOptions()); + Response getResponse = restClient.performRequest(get); + assertThat(getResponse.getStatusLine().getStatusCode(), is(200)); + + // search all + Request search = new Request("GET", "/_search"); + search.setJsonEntity("{ \"query\": { \"match_all\": {} } }"); + + // no header + Response searchResponse = restClient.performRequest(search); + assertThat(searchResponse.getStatusLine().getStatusCode(), is(200)); + responseMap = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + EntityUtils.toString(searchResponse.getEntity()), + false + ); + Map hits = (Map) responseMap.get("hits"); + List hitsHits = (List) hits.get("hits"); + assertThat(hitsHits.size(), is(0)); + + // wrong header + search.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "wrong").build()); + searchResponse = restClient.performRequest(search); + assertThat(searchResponse.getStatusLine().getStatusCode(), is(200)); + responseMap = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + EntityUtils.toString(searchResponse.getEntity()), + false + ); + hits = (Map) responseMap.get("hits"); + hitsHits = (List) hits.get("hits"); + assertThat(hitsHits.size(), is(0)); + + // correct + search.setOptions(putRequest.getOptions()); + searchResponse = restClient.performRequest(search); + assertThat(searchResponse.getStatusLine().getStatusCode(), is(200)); + responseMap = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + EntityUtils.toString(searchResponse.getEntity()), + false + ); + hits = (Map) responseMap.get("hits"); + hitsHits = (List) hits.get("hits"); + assertThat(hitsHits.size(), is(1)); + + // search the datastream + Request searchIdx = new Request("GET", "/.test-data-stream/_search"); + searchIdx.setJsonEntity("{ \"query\": { \"match_all\": {} } }"); + + // no header + re = expectThrows(ResponseException.class, () -> restClient.performRequest(searchIdx)); + assertThat(re.getMessage(), containsString("reserved for system")); + + // incorrect header + searchIdx.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "wrong").build()); + re = expectThrows(ResponseException.class, () -> restClient.performRequest(searchIdx)); + assertThat(re.getMessage(), containsString("accessed by product")); + + // correct + searchIdx.setOptions(putRequest.getOptions()); + searchResponse = restClient.performRequest(searchIdx); + assertThat(searchResponse.getStatusLine().getStatusCode(), is(200)); + responseMap = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + EntityUtils.toString(searchResponse.getEntity()), + false + ); + hits = (Map) responseMap.get("hits"); + hitsHits = (List) hits.get("hits"); + assertThat(hitsHits.size(), is(1)); + } + } + + @After + public void cleanup() { + try { + PlainActionFuture stateStatusPlainActionFuture = new PlainActionFuture<>(); + new TestSystemDataStreamPlugin().cleanUpFeature( + internalCluster().clusterService(), + internalCluster().client(), + stateStatusPlainActionFuture + ); + stateStatusPlainActionFuture.actionGet(); + } catch (ResourceNotFoundException e) { + // ignore + } + } + + public static final class TestSystemDataStreamPlugin extends Plugin implements SystemIndexPlugin { + + @Override + public Collection getSystemDataStreamDescriptors() { + try { + CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}"); + return List.of( + new SystemDataStreamDescriptor( + ".test-data-stream", + "system data stream test", + Type.EXTERNAL, + new ComposableIndexTemplate( + List.of(".test-data-stream"), + new Template(Settings.EMPTY, mappings, null), + null, + null, + null, + null, + new DataStreamTemplate() + ), + Map.of(), + List.of("product") + ) + ); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public String getFeatureName() { + return SystemDataStreamIT.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "Integration testing of system data streams"; + } + + @Override + public void cleanUpFeature(ClusterService clusterService, Client client, ActionListener listener) { + Collection dataStreamDescriptors = getSystemDataStreamDescriptors(); + final DeleteDataStreamAction.Request request = new DeleteDataStreamAction.Request( + dataStreamDescriptors.stream() + .map(SystemDataStreamDescriptor::getDataStreamName) + .collect(Collectors.toList()) + .toArray(Strings.EMPTY_ARRAY) + ); + EnumSet