[7.x] Add include_data_streams flag for authorization

modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexValidator.java

@@ -120,9 +120,9 @@ static void validateAgainstAliases(SearchRequest source, IndexRequest destinatio
              * it. This is the same sort of dance that TransportIndexRequest
              * uses to decide to autocreate the index.
              */
-            target = indexNameExpressionResolver.concreteWriteIndex(clusterState, destination, true).getName();
+            target = indexNameExpressionResolver.concreteWriteIndex(clusterState, destination).getName();
         }
-        for (String sourceIndex : indexNameExpressionResolver.concreteIndexNames(clusterState, source, true)) {
+        for (String sourceIndex : indexNameExpressionResolver.concreteIndexNames(clusterState, source)) {
             if (sourceIndex.equals(target)) {
                 ActionRequestValidationException e = new ActionRequestValidationException();
                 e.addValidationError("reindex cannot write into an index its reading from [" + target + ']');

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml

@@ -49,3 +49,14 @@
       indices.delete_data_stream:
         name: logs-foobar
   - is_true: acknowledged
+
+---
+"APIs temporarily muted":
+  - skip:
+      version: "all"
+      reason: "restore to above test after data stream resolution PRs have been merged"
+
+  - do:
+      catch: bad_request
+      indices.close:
+        index: logs-*

server/src/main/java/org/elasticsearch/action/IndicesRequest.java

@@ -24,7 +24,7 @@
 /**
  * Needs to be implemented by all {@link org.elasticsearch.action.ActionRequest} subclasses that relate to
  * one or more indices. Allows to retrieve which indices the action relates to.
- * In case of internal requests  originated during the distributed execution of an external request,
+ * In case of internal requests originated during the distributed execution of an external request,
  * they will still return the indices that the original request related to.
  */
 public interface IndicesRequest {

server/src/main/java/org/elasticsearch/action/OriginalIndices.java

@@ -56,6 +56,11 @@ public IndicesOptions indicesOptions() {
         return indicesOptions;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     public static OriginalIndices readOriginalIndices(StreamInput in) throws IOException {
         return new OriginalIndices(in.readStringArray(), IndicesOptions.readIndicesOptions(in));
     }

server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java

@@ -147,6 +147,11 @@ public ClusterHealthRequest indicesOptions(final IndicesOptions indicesOptions)
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     public TimeValue timeout() {
         return timeout;
     }

server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java

@@ -300,7 +300,7 @@ static int prepareResponse(final ClusterHealthRequest request, final ClusterHeal
         }
         if (CollectionUtils.isEmpty(request.indices()) == false) {
             try {
-                indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.strictExpand(), true, request.indices());
+                indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.strictExpand(), request);
                 waitForCounter++;
             } catch (IndexNotFoundException e) {
                 response.setStatus(ClusterHealthStatus.RED); // no indices, make sure its RED
@@ -367,7 +367,7 @@ private ClusterHealthResponse clusterHealth(ClusterHealthRequest request, Cluste
 
         String[] concreteIndices;
         try {
-            concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request, true);
+            concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         } catch (IndexNotFoundException e) {
             // one of the specified indices is not there - treat it as RED.
             ClusterHealthResponse response = new ClusterHealthResponse(clusterState.getClusterName().value(), Strings.EMPTY_ARRAY,

server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java

@@ -68,7 +68,7 @@ protected String executor() {
     @Override
     protected ClusterBlockException checkBlock(ClusterSearchShardsRequest request, ClusterState state) {
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
-                indexNameExpressionResolver.concreteIndexNames(state, request, true));
+                indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -80,7 +80,7 @@ protected ClusterSearchShardsResponse read(StreamInput in) throws IOException {
     protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state,
                                    final ActionListener<ClusterSearchShardsResponse> listener) {
         ClusterState clusterState = clusterService.state();
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request, true);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices());
         Map<String, AliasFilter> indicesAndFilters = new HashMap<>();
         Set<String> indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices());

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java

@@ -275,6 +275,10 @@ public CreateSnapshotRequest indicesOptions(IndicesOptions indicesOptions) {
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
 
     /**
      * Returns true if indices with unavailable shards should be be partially snapshotted.

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java

@@ -161,6 +161,11 @@ public final ClusterStateRequest indicesOptions(IndicesOptions indicesOptions) {
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     public ClusterStateRequest customs(boolean customs) {
         this.customs = customs;
         return this;

server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

@@ -144,7 +144,7 @@ private ClusterStateResponse buildResponse(final ClusterStateRequest request,
         if (request.routingTable()) {
             if (request.indices().length > 0) {
                 RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
-                String[] indices = indexNameExpressionResolver.concreteIndexNames(currentState, request, true);
+                String[] indices = indexNameExpressionResolver.concreteIndexNames(currentState, request);
                 for (String filteredIndex : indices) {
                     if (currentState.routingTable().getIndicesRouting().containsKey(filteredIndex)) {
                         routingTableBuilder.add(currentState.routingTable().getIndicesRouting().get(filteredIndex));
@@ -166,7 +166,7 @@ private ClusterStateResponse buildResponse(final ClusterStateRequest request,
         if (request.metadata()) {
             if (request.indices().length > 0) {
                 mdBuilder.version(currentState.metadata().version());
-                String[] indices = indexNameExpressionResolver.concreteIndexNames(currentState, request, true);
+                String[] indices = indexNameExpressionResolver.concreteIndexNames(currentState, request);
                 for (String filteredIndex : indices) {
                     IndexMetadata indexMetadata = currentState.metadata().index(filteredIndex);
                     if (indexMetadata != null) {

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java

@@ -103,6 +103,11 @@ public IndicesOptions indicesOptions() {
         return indicesOptions;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     /** @param fields a list of fields to retrieve the mapping for */
     public GetFieldMappingsRequest fields(String... fields) {
         this.fields = fields;

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java

@@ -56,7 +56,7 @@ public TransportGetFieldMappingsAction(TransportService transportService, Cluste
     @Override
     protected void doExecute(Task task, GetFieldMappingsRequest request, final ActionListener<GetFieldMappingsResponse> listener) {
         ClusterState clusterState = clusterService.state();
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request, true);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         final AtomicInteger indexCounter = new AtomicInteger();
         final AtomicInteger completionCounter = new AtomicInteger(concreteIndices.length);
         final AtomicReferenceArray<Object> indexResponses = new AtomicReferenceArray<>(concreteIndices.length);

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java

@@ -169,6 +169,11 @@ public PutMappingRequest indicesOptions(IndicesOptions indicesOptions) {
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     /**
      * The mapping type.
      */

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java

@@ -86,7 +86,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException {
     protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterState state) {
         String[] indices;
         if (request.getConcreteIndex() == null) {
-            indices = indexNameExpressionResolver.concreteIndexNames(state, request, true);
+            indices = indexNameExpressionResolver.concreteIndexNames(state, request);
         } else {
             indices = new String[] {request.getConcreteIndex().getName()};
         }
@@ -98,7 +98,7 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta
                                    final ActionListener<AcknowledgedResponse> listener) {
         try {
             final Index[] concreteIndices = request.getConcreteIndex() == null ?
-                indexNameExpressionResolver.concreteIndices(state, request, true)
+                indexNameExpressionResolver.concreteIndices(state, request)
                 : new Index[] {request.getConcreteIndex()};
             final Optional<Exception> maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices);
             if (maybeValidationException.isPresent()) {

server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java

@@ -113,6 +113,11 @@ public OpenIndexRequest indicesOptions(IndicesOptions indicesOptions) {
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     public ActiveShardCount waitForActiveShards() {
         return waitForActiveShards;
     }

server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java

@@ -83,13 +83,13 @@ protected void doExecute(Task task, OpenIndexRequest request, ActionListener<Ope
     @Override
     protected ClusterBlockException checkBlock(OpenIndexRequest request, ClusterState state) {
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE,
-            indexNameExpressionResolver.concreteIndexNames(state, request, true));
+            indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
     protected void masterOperation(final OpenIndexRequest request, final ClusterState state,
                                    final ActionListener<OpenIndexResponse> listener) {
-        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request, true);
+        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         if (concreteIndices == null || concreteIndices.length == 0) {
             listener.onResponse(new OpenIndexResponse(true, true));
             return;

server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java

@@ -140,6 +140,11 @@ public IndicesRequest indices(String... indices) {
             this.names = indices;
             return this;
         }
+
+        @Override
+        public boolean includeDataStreams() {
+            return true;
+        }
     }
 
     public static class ResolvedIndexAbstraction {

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java

@@ -151,6 +151,11 @@ public IndicesOptions indicesOptions() {
         return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     /**
      * Sets the rollover target to rollover to another index
      */

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java

@@ -91,7 +91,7 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState
             request.indicesOptions().expandWildcardsOpen(), request.indicesOptions().expandWildcardsClosed());
 
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE,
-            indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, true, request.indices()));
+            indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, request));
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequest.java

@@ -61,7 +61,6 @@ public GetSettingsRequest includeDefaults(boolean includeDefaults) {
         return this;
     }
 
-
     public GetSettingsRequest() {
     }
 
@@ -98,6 +97,11 @@ public IndicesOptions indicesOptions() {
         return indicesOptions;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     public String[] names() {
         return names;
     }

server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java

@@ -68,7 +68,7 @@ protected String executor() {
     @Override
     protected ClusterBlockException checkBlock(GetSettingsRequest request, ClusterState state) {
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
-            indexNameExpressionResolver.concreteIndexNames(state, request, true));
+            indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
 
@@ -83,7 +83,7 @@ private static boolean isFilteredRequest(GetSettingsRequest request) {
 
     @Override
     protected void masterOperation(GetSettingsRequest request, ClusterState state, ActionListener<GetSettingsResponse> listener) {
-        Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request, true);
+        Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         ImmutableOpenMap.Builder<String, Settings> indexToSettingsBuilder = ImmutableOpenMap.builder();
         ImmutableOpenMap.Builder<String, Settings> indexToDefaultSettingsBuilder = ImmutableOpenMap.builder();
         for (Index concreteIndex : concreteIndices) {

server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java

@@ -77,7 +77,7 @@ protected ClusterBlockException checkBlock(UpdateSettingsRequest request, Cluste
             return null;
         }
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE,
-            indexNameExpressionResolver.concreteIndexNames(state, request, true));
+            indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -88,7 +88,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException {
     @Override
     protected void masterOperation(final UpdateSettingsRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
-        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request, true);
+        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest()
                 .indices(concreteIndices)
                 .settings(request.settings())

server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java

@@ -119,6 +119,11 @@ public UpdateSettingsRequest indicesOptions(IndicesOptions indicesOptions) {
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     /**
      * Sets the settings to be updated
      */

server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java

@@ -726,7 +726,8 @@ Index resolveIfAbsent(DocWriteRequest<?> request) {
             Index concreteIndex = indices.get(request.index());
             if (concreteIndex == null) {
                 boolean includeDataStreams = request.opType() == DocWriteRequest.OpType.CREATE;
-                concreteIndex = indexNameExpressionResolver.concreteWriteIndex(state, request, includeDataStreams);
+                concreteIndex = indexNameExpressionResolver.concreteWriteIndex(state, request.indicesOptions(), request.indices()[0],
+                    false, includeDataStreams);
                 indices.put(request.index(), concreteIndex);
             }
             return concreteIndex;

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -78,7 +78,7 @@ protected void doExecute(Task task, FieldCapabilitiesRequest request, final Acti
             // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices
             concreteIndices = Strings.EMPTY_ARRAY;
         } else {
-            concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, localIndices, true);
+            concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, localIndices);
         }
         final int totalNumRequest = concreteIndices.length + remoteClusterIndices.size();
         final CountDown completionCounter = new CountDown(totalNumRequest);

server/src/main/java/org/elasticsearch/action/index/IndexRequest.java

@@ -778,6 +778,10 @@ public String toString() {
         return "index {[" + index + "][" + type() + "][" + id + "], source[" + sSource + "]}";
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
 
     /**
      * Returns <code>true</code> if this request has been sent to a shard copy more than once.

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -451,14 +451,12 @@ static BiFunction<String, String, DiscoveryNode> processRemoteShards(Map<String,
     }
 
     private Index[] resolveLocalIndices(OriginalIndices localIndices,
-                                IndicesOptions indicesOptions,
                                 ClusterState clusterState,
                                 SearchTimeProvider timeProvider) {
         if (localIndices == null) {
             return Index.EMPTY_ARRAY; //don't search on any local index (happens when only remote indices were specified)
         }
-        return indexNameExpressionResolver.concreteIndices(clusterState, indicesOptions, true,
-            timeProvider.getAbsoluteStartMillis(), localIndices.indices());
+        return indexNameExpressionResolver.concreteIndices(clusterState, localIndices, timeProvider.getAbsoluteStartMillis());
     }
 
     private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, SearchRequest searchRequest,
@@ -471,7 +469,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
         // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
         // of just for the _search api
-        final Index[] indices = resolveLocalIndices(localIndices, searchRequest.indicesOptions(), clusterState, timeProvider);
+        final Index[] indices = resolveLocalIndices(localIndices, clusterState, timeProvider);
         Map<String, AliasFilter> aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices, remoteAliasMap);
         Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(),
             searchRequest.indices());

server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java

@@ -76,6 +76,11 @@ public final Request indicesOptions(IndicesOptions indicesOptions) {
         return (Request) this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);