Merge branch 'master' into modules/mapper_extras

* master: (21 commits)
  Ensure module is bundled before installing in tests
  Add boolean similarity to built in similarity types (elastic#26613)
  [Tests] Remove skip tests in search/30_limits.yml
  Let search phases override max concurrent requests
  Add a soft limit for the number of requested doc-value fields (elastic#26574)
  Support for accessing Azure repositories through a proxy (elastic#23518)
  Add beta tag to MSI Windows Installer (elastic#26616)
  Fix Lucene version of 5.6.1.
  Remove azure deprecated settings (elastic#26099)
  Handle the 5.6.0 release
  Allow plugins to validate cluster-state on join (elastic#26595)
  Remove index mapper dynamic settings (elastic#25734)
  update AWS SDK for ECS Task IAM support in discovery-ec2 (elastic#26479)
  Azure repository: Accelerate the listing of files (used in delete snapshot) (elastic#25710)
  Build: Remove norelease from forbidden patterns (elastic#26592)
  Fix reference to painless inside expression engine (elastic#26528)
  Build: Move javadoc linking to root build.gradle (elastic#26529)
  Test: Remove leftover static bwc test case (elastic#26584)
  Docs: Remove remaining references to file and native scripts (elastic#26580)
  Snapshot fallback should consider build.snapshot
  ...

build.gradle

@@ -204,25 +204,15 @@ task branchConsistency {
 }
 
 subprojects {
-  project.afterEvaluate {
-    // ignore missing javadocs
-    tasks.withType(Javadoc) { Javadoc javadoc ->
-      // the -quiet here is because of a bug in gradle, in that adding a string option
-      // by itself is not added to the options. By adding quiet, both this option and
-      // the "value" -quiet is added, separated by a space. This is ok since the javadoc
-      // command already adds -quiet, so we are just duplicating it
-      // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
-      javadoc.options.encoding='UTF8'
-      javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
-      /*
-      TODO: building javadocs with java 9 b118 is currently broken with weird errors, so
-      for now this is commented out...try again with the next ea build...
-      javadoc.executable = new File(project.javaHome, 'bin/javadoc')
-      if (project.javaVersion == JavaVersion.VERSION_1_9) {
-        // TODO: remove this hack! gradle should be passing this...
-        javadoc.options.addStringOption('source', '8')
-      }*/
-    }
+  // ignore missing javadocs
+  tasks.withType(Javadoc) { Javadoc javadoc ->
+    // the -quiet here is because of a bug in gradle, in that adding a string option
+    // by itself is not added to the options. By adding quiet, both this option and
+    // the "value" -quiet is added, separated by a space. This is ok since the javadoc
+    // command already adds -quiet, so we are just duplicating it
+    // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
+    javadoc.options.encoding='UTF8'
+    javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
   }
 
   /* Sets up the dependencies that we build as part of this project but
@@ -280,6 +270,27 @@ subprojects {
         }
       }
     }
+
+    // Handle javadoc dependencies across projects. Order matters: the linksOffline for
+    // org.elasticsearch:elasticsearch must be the last one or all the links for the
+    // other packages (e.g org.elasticsearch.client) will point to core rather than
+    // their own artifacts.
+    if (project.plugins.hasPlugin(BuildPlugin)) {
+      String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
+      Closure sortClosure = { a, b -> b.group <=> a.group }
+      Closure depJavadocClosure = { dep ->
+        if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
+          String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
+          if (substitution != null) {
+            project.javadoc.dependsOn substitution + ':javadoc'
+            String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
+            project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
+          }
+        }
+      }
+      project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+      project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+    }
   }
 }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -455,28 +455,8 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     static void configureJavadoc(Project project) {
-        String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
-        project.afterEvaluate {
-            project.tasks.withType(Javadoc) {
-                executable = new File(project.javaHome, 'bin/javadoc')
-            }
-            /*
-             * Order matters, the linksOffline for org.elasticsearch:elasticsearch must be the last one
-             * or all the links for the other packages (e.g org.elasticsearch.client) will point to core rather than their own artifacts
-             */
-            Closure sortClosure = { a, b -> b.group <=> a.group }
-            Closure depJavadocClosure = { dep ->
-                if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
-                    String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
-                    if (substitution != null) {
-                        project.javadoc.dependsOn substitution + ':javadoc'
-                        String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
-                        project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
-                    }
-                }
-            }
-            project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
-            project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+        project.tasks.withType(Javadoc) {
+            executable = new File(project.javaHome, 'bin/javadoc')
         }
         configureJavadocJar(project)
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy

@@ -63,11 +63,6 @@ public class ForbiddenPatternsTask extends DefaultTask {
         patterns.put('nocommit should be all lowercase or all uppercase',
             /((?i)nocommit)(?<!(nocommit|NOCOMMIT))/)
         patterns.put('tab', /\t/)
-        if (System.getProperty('build.snapshot', 'true').equals('false')) {
-            patterns.put('norelease', /norelease|NORELEASE/)
-        }
-        patterns.put('norelease should be all lowercase or all uppercase',
-            /((?i)norelease)(?<!(norelease|NORELEASE))/)
 
 
         inputs.property("excludes", filesFilter.excludes)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -489,6 +489,7 @@ class ClusterFormationTasks {
         }
         Copy installModule = project.tasks.create(name, Copy.class)
         installModule.dependsOn(setup)
+        installModule.dependsOn(module.tasks.bundlePlugin)
         installModule.into(new File(node.homeDir, "modules/${module.name}"))
         installModule.from({ project.zipTree(module.tasks.bundlePlugin.outputs.files.singleFile) })
         return installModule

core/src/main/java/org/elasticsearch/Build.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch;
 
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -59,7 +60,18 @@ public class Build {
             // not running from the official elasticsearch jar file (unit tests, IDE, uber client jar, shadiness)
             shortHash = "Unknown";
             date = "Unknown";
-            isSnapshot = true;
+            final String buildSnapshot = System.getProperty("build.snapshot");
+            if (buildSnapshot != null) {
+                try {
+                    Class.forName("com.carrotsearch.randomizedtesting.RandomizedContext");
+                } catch (final ClassNotFoundException e) {
+                    // we are not in tests but build.snapshot is set, bail hard
+                    throw new IllegalStateException("build.snapshot set to [" + buildSnapshot + "] but not running tests");
+                }
+                isSnapshot = Booleans.parseBoolean(buildSnapshot);
+            } else {
+                isSnapshot = true;
+            }
         }
         if (shortHash == null) {
             throw new IllegalStateException("Error finding the build shortHash. " +

core/src/main/java/org/elasticsearch/Version.java

@@ -92,6 +92,9 @@ public class Version implements Comparable<Version> {
     public static final Version V_5_5_3 = new Version(V_5_5_3_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
     public static final int V_5_6_0_ID = 5060099;
     public static final Version V_5_6_0 = new Version(V_5_6_0_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
+    public static final int V_5_6_1_ID = 5060199;
+    // use proper Lucene constant once we are on a Lucene snapshot that knows about 6.6.1
+    public static final Version V_5_6_1 = new Version(V_5_6_1_ID, org.apache.lucene.util.Version.fromBits(6, 6, 1));
     public static final int V_6_0_0_alpha1_ID = 6000001;
     public static final Version V_6_0_0_alpha1 =
             new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
@@ -142,6 +145,8 @@ public static Version fromId(int id) {
                 return V_6_0_0_alpha2;
             case V_6_0_0_alpha1_ID:
                 return V_6_0_0_alpha1;
+            case V_5_6_1_ID:
+                return V_5_6_1;
             case V_5_6_0_ID:
                 return V_5_6_0;
             case V_5_5_3_ID:

core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java

@@ -76,8 +76,8 @@ protected AbstractSearchAsyncAction(String name, Logger logger, SearchTransportS
                                         Executor executor, SearchRequest request,
                                         ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
                                         TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion,
-                                        SearchTask task, SearchPhaseResults<Result> resultConsumer) {
-        super(name, request, shardsIts, logger);
+                                        SearchTask task, SearchPhaseResults<Result> resultConsumer, int maxConcurrentShardRequests) {
+        super(name, request, shardsIts, logger, maxConcurrentShardRequests);
         this.timeProvider = timeProvider;
         this.logger = logger;
         this.searchTransportService = searchTransportService;

core/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java

@@ -26,10 +26,6 @@
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.transport.Transport;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
@@ -55,9 +51,12 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction<Searc
                                         ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
                                         TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion,
                                         SearchTask task, Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory) {
+        /*
+         * We set max concurrent shard requests to the number of shards to otherwise avoid deep recursing that would occur if the local node
+         * is the coordinating node for the query, holds all the shards for the request, and there are a lot of shards.
+         */
         super("can_match", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, executor, request,
-            listener,
-            shardsIts, timeProvider, clusterStateVersion, task, new BitSetSearchPhaseResults(shardsIts.size()));
+            listener, shardsIts, timeProvider, clusterStateVersion, task, new BitSetSearchPhaseResults(shardsIts.size()), shardsIts.size());
         this.phaseFactory = phaseFactory;
         this.shardsIts = shardsIts;
     }

core/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java

@@ -52,7 +52,8 @@ abstract class InitialSearchPhase<FirstResult extends SearchPhaseResult> extends
     private final AtomicInteger shardExecutionIndex = new AtomicInteger(0);
     private final int maxConcurrentShardRequests;
 
-    InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger) {
+    InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger,
+                       int maxConcurrentShardRequests) {
         super(name);
         this.request = request;
         this.shardsIts = shardsIts;
@@ -62,7 +63,7 @@ abstract class InitialSearchPhase<FirstResult extends SearchPhaseResult> extends
         // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result
         // we process hence we add one for the non active partition here.
         this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
-        maxConcurrentShardRequests = Math.min(request.getMaxConcurrentShardRequests(), shardsIts.size());
+        this.maxConcurrentShardRequests = Math.min(maxConcurrentShardRequests, shardsIts.size());
     }
 
     private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId,

core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java

@@ -42,7 +42,8 @@ final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction
             final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider,
             final long clusterStateVersion, final SearchTask task) {
         super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, executor, request, listener,
-                shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()));
+                shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()),
+                request.getMaxConcurrentShardRequests());
         this.searchPhaseController = searchPhaseController;
     }
 

core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java

@@ -42,7 +42,8 @@ final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<Se
             final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider,
             long clusterStateVersion, SearchTask task) {
         super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, executor, request, listener,
-            shardsIts, timeProvider, clusterStateVersion, task, searchPhaseController.newSearchPhaseResults(request, shardsIts.size()));
+            shardsIts, timeProvider, clusterStateVersion, task, searchPhaseController.newSearchPhaseResults(request, shardsIts.size()),
+                request.getMaxConcurrentShardRequests());
         this.searchPhaseController = searchPhaseController;
     }
 

core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java

@@ -47,9 +47,9 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.TaskAwareTransportRequestHandler;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportActionProxy;
-import org.elasticsearch.transport.TaskAwareTransportRequestHandler;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
@@ -59,7 +59,6 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.BiFunction;
@@ -447,7 +446,7 @@ public void messageReceived(ShardFetchSearchRequest request, TransportChannel ch
             });
         TransportActionProxy.registerProxyAction(transportService, FETCH_ID_ACTION_NAME, FetchSearchResult::new);
 
-        // this is super cheap and should not hit thread-pool rejections
+        // this is cheap, it does not fetch during the rewrite phase, so we can let it quickly execute on a networking thread
         transportService.registerRequestHandler(QUERY_CAN_MATCH_NAME, ThreadPool.Names.SAME, ShardSearchTransportRequest::new,
             new TaskAwareTransportRequestHandler<ShardSearchTransportRequest>() {
                 @Override

core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -111,6 +111,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
         IndexSettings.MAX_RESULT_WINDOW_SETTING,
         IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING,
+        IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING,
         IndexSettings.MAX_RESCORE_WINDOW_SETTING,
         IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
         IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,

core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.discovery;
 
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterApplier;
 import org.elasticsearch.cluster.service.MasterService;
@@ -36,12 +38,15 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
@@ -62,14 +67,18 @@ public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportServic
                            ClusterApplier clusterApplier, ClusterSettings clusterSettings, List<DiscoveryPlugin> plugins,
                            AllocationService allocationService) {
         final UnicastHostsProvider hostsProvider;
-
+        final Collection<BiConsumer<DiscoveryNode,ClusterState>> joinValidators = new ArrayList<>();
         Map<String, Supplier<UnicastHostsProvider>> hostProviders = new HashMap<>();
         for (DiscoveryPlugin plugin : plugins) {
             plugin.getZenHostsProviders(transportService, networkService).entrySet().forEach(entry -> {
                 if (hostProviders.put(entry.getKey(), entry.getValue()) != null) {
                     throw new IllegalArgumentException("Cannot register zen hosts provider [" + entry.getKey() + "] twice");
                 }
             });
+            BiConsumer<DiscoveryNode, ClusterState> joinValidator = plugin.getJoinValidator();
+            if (joinValidator != null) {
+                joinValidators.add(joinValidator);
+            }
         }
         Optional<String> hostsProviderName = DISCOVERY_HOSTS_PROVIDER_SETTING.get(settings);
         if (hostsProviderName.isPresent()) {
@@ -85,7 +94,7 @@ public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportServic
         Map<String, Supplier<Discovery>> discoveryTypes = new HashMap<>();
         discoveryTypes.put("zen",
             () -> new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, masterService, clusterApplier,
-                clusterSettings, hostsProvider, allocationService));
+                clusterSettings, hostsProvider, allocationService, Collections.unmodifiableCollection(joinValidators)));
         discoveryTypes.put("single-node", () -> new SingleNodeDiscovery(settings, transportService, masterService, clusterApplier));
         for (DiscoveryPlugin plugin : plugins) {
             plugin.getDiscoveryTypes(threadPool, transportService, namedWriteableRegistry,