Merge branch 'master' into compile-with-jdk-9

* master: (30 commits)
  Fix lock accounting in releasable lock
  Add ability to associate an ID with tasks  (elastic#27764)
  [DOCS] Removed differencies between text and code (elastic#27993)
  text fixes (elastic#28136)
  Update getting-started.asciidoc (elastic#28145)
  [Docs] Spelling fix in painless-getting-started.asciidoc (elastic#28187)
  Fixed the cat.health REST test to accept 4ms, not just 4.0ms (elastic#28186)
  Do not keep 5.x commits once having 6.x commits (elastic#28188)
  Rename core module to server (elastic#28180)
  upgraded jna from 4.4.0-1 to 4.5.1 (elastic#28183)
  [TEST] Do not call RandomizedTest.scaledRandomIntBetween from multiple threads
  Primary send safe commit in file-based recovery (elastic#28038)
  [Docs] Correct response json in rank-eval.asciidoc
  Add scroll parameter to _reindex API (elastic#28041)
  Include all sentences smaller than fragment_size in the unified highlighter (elastic#28132)
  Modifies the JavaAPI docs related to AggregationBuilder
  [Docs] Improvements in script-fields.asciidoc (elastic#28174)
  [Docs] Remove Kerberos/SPNEGO Shield plugin (elastic#28019)
  Ignore null value for range field (elastic#27845) (elastic#28116)
  Fix environment variable substitutions in list setting (elastic#28106)
  ...

build.gradle

@@ -79,7 +79,7 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
  * backwards compatibility guarantees and only keeping the latest beta or rc
  * in a branch if there are only betas and rcs in the branch so we have
  * *something* to test against. */
-VersionCollection versions = new VersionCollection(file('core/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
+VersionCollection versions = new VersionCollection(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
 if (versions.currentVersion.toString() != VersionProperties.elasticsearch) {
   throw new GradleException("The last version in Versions.java [${versions.currentVersion}] does not match " +
           "VersionProperties.elasticsearch [${VersionProperties.elasticsearch}]")
@@ -181,8 +181,8 @@ subprojects {
   ext.projectSubstitutions = [
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
-    "org.elasticsearch:elasticsearch:${version}": ':core',
-    "org.elasticsearch:elasticsearch-cli:${version}": ':core:cli',
+    "org.elasticsearch:elasticsearch:${version}": ':server',
+    "org.elasticsearch:elasticsearch-cli:${version}": ':server:cli',
     "org.elasticsearch:elasticsearch-nio:${version}": ':libs:elasticsearch-nio',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
@@ -227,7 +227,7 @@ subprojects {
 
     // Handle javadoc dependencies across projects. Order matters: the linksOffline for
     // org.elasticsearch:elasticsearch must be the last one or all the links for the
-    // other packages (e.g org.elasticsearch.client) will point to core rather than
+    // other packages (e.g org.elasticsearch.client) will point to server rather than
     // their own artifacts.
     if (project.plugins.hasPlugin(BuildPlugin)) {
       String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
@@ -255,7 +255,7 @@ subprojects {
 gradle.projectsEvaluated {
   allprojects {
     if (project.path == ':test:framework') {
-      // :test:framework:test cannot run before and after :core:test
+      // :test:framework:test cannot run before and after :server:test
       return
     }
     configurations.all {

buildSrc/build.gradle

@@ -96,7 +96,7 @@ dependencies {
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
   compile 'de.thetaphi:forbiddenapis:2.4.1'
   compile 'org.apache.rat:apache-rat:0.11'
-  compile "org.elasticsearch:jna:4.4.0-1"
+  compile "org.elasticsearch:jna:4.5.1"
 }
 
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -32,6 +32,7 @@ import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.XmlProvider
 import org.gradle.api.artifacts.Configuration
+import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.ModuleDependency
 import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
@@ -280,8 +281,9 @@ class BuildPlugin implements Plugin<Project> {
         })
 
         // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
-        Closure disableTransitiveDeps = { ModuleDependency dep ->
-            if (!(dep instanceof ProjectDependency) && dep.group.startsWith('org.elasticsearch') == false) {
+        Closure disableTransitiveDeps = { Dependency dep ->
+            if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency)
+                    && dep.group.startsWith('org.elasticsearch') == false) {
                 dep.transitive = false
 
                 // also create a configuration just for this dependency version, so that later

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -84,7 +84,7 @@ class PrecommitTasks {
         Task mainForbidden = project.tasks.findByName('forbiddenApisMain')
         if (mainForbidden != null) {
             mainForbidden.configure {
-                signaturesURLs += getClass().getResource('/forbidden/es-core-signatures.txt')
+                signaturesURLs += getClass().getResource('/forbidden/es-server-signatures.txt')
             }
         }
         Task testForbidden = project.tasks.findByName('forbiddenApisTest')

buildSrc/version.properties

@@ -11,21 +11,21 @@ log4j             = 2.9.1
 slf4j             = 1.6.2
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
-jna               = 4.4.0-1
+jna               = 4.5.1
 
 # test dependencies
 randomizedrunner  = 2.5.2
 junit             = 4.12
 httpclient        = 4.5.2
-# When updating httpcore, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 httpcore          = 4.4.5
-# When updating httpasyncclient, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+# When updating httpasyncclient, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 httpasyncclient   = 4.1.2
 commonslogging    = 1.1.3
 commonscodec      = 1.10
 hamcrest          = 1.3
 securemock        = 1.2
-# When updating mocksocket, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+# When updating mocksocket, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 mocksocket        = 1.2
 
 # benchmark dependencies

client/rest/build.gradle

@@ -57,7 +57,7 @@ dependencies {
 }
 
 forbiddenApisMain {
-  //client does not depend on core, so only jdk and http signatures should be checked
+  //client does not depend on server, so only jdk and http signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt'),
                     PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
 }
@@ -66,12 +66,12 @@ forbiddenApisTest {
   //we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
   bundledSignatures -= 'jdk-non-portable'
   bundledSignatures += 'jdk-internal'
-  //client does not depend on core, so only jdk signatures should be checked
+  //client does not depend on server, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt'),
                     PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
 }
 
-//JarHell is part of es core, which we don't want to pull in
+// JarHell is part of es server, which we don't want to pull in
 jarHell.enabled=false
 
 namingConventions {

client/sniffer/build.gradle

@@ -57,15 +57,15 @@ dependencies {
 }
 
 forbiddenApisMain {
-  //client does not depend on core, so only jdk signatures should be checked
+  //client does not depend on server, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
 forbiddenApisTest {
   //we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
   bundledSignatures -= 'jdk-non-portable'
   bundledSignatures += 'jdk-internal'
-  //client does not depend on core, so only jdk signatures should be checked
+  //client does not depend on server, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
@@ -74,7 +74,7 @@ dependencyLicenses {
   mapping from: /commons-.*/, to: 'commons'
 }
 
-//JarHell is part of es core, which we don't want to pull in
+// JarHell is part of es server, which we don't want to pull in
 jarHell.enabled=false
 
 namingConventions {
@@ -101,4 +101,4 @@ thirdPartyAudit.excludes = [
   //commons-logging provided dependencies
   'javax.servlet.ServletContextEvent',
   'javax.servlet.ServletContextListener'
-]
+]

client/test/build.gradle

@@ -48,7 +48,7 @@ forbiddenApisTest {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-//JarHell is part of es core, which we don't want to pull in
+// JarHell is part of es server, which we don't want to pull in
 jarHell.enabled=false
 
 // TODO: should we have licenses for our test deps?

distribution/build.gradle

@@ -59,14 +59,14 @@ task generateDependenciesReport(type: ConcatFilesTask) {
  *                                Notice file                                *
  *****************************************************************************/
 
-// integ test zip only uses core, so a different notice file is needed there
+// integ test zip only uses server, so a different notice file is needed there
 task buildCoreNotice(type: NoticeTask) {
-  licensesDir new File(project(':core').projectDir, 'licenses')
+  licensesDir new File(project(':server').projectDir, 'licenses')
 }
 
 // other distributions include notices from modules as well, which are added below later
 task buildFullNotice(type: NoticeTask) {
-  licensesDir new File(project(':core').projectDir, 'licenses')
+  licensesDir new File(project(':server').projectDir, 'licenses')
 }
 
 /*****************************************************************************
@@ -173,8 +173,8 @@ configure(distributions) {
      *****************************************************************************/
     libFiles = copySpec {
       into 'lib'
-      from project(':core').jar
-      from project(':core').configurations.runtime
+      from { project(':server').jar }
+      from { project(':server').configurations.runtime }
       from { project(':libs:plugin-classloader').jar }
       // delay add tools using closures, since they have not yet been configured, so no jar task exists yet
       from { project(':distribution:tools:launchers').jar }

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -232,14 +232,14 @@ private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Ex
 
         // now try as maven coordinates, a valid URL would only have a colon and slash
         String[] coordinates = pluginId.split(":");
-        if (coordinates.length == 3 && pluginId.contains("/") == false) {
+        if (coordinates.length == 3 && pluginId.contains("/") == false && pluginId.startsWith("file:") == false) {
             String mavenUrl = getMavenUrl(terminal, coordinates, Platforms.PLATFORM_NAME);
             terminal.println("-> Downloading " + pluginId + " from maven central");
             return downloadZipAndChecksum(terminal, mavenUrl, tmpDir, true);
         }
 
         // fall back to plain old URL
-        if (pluginId.contains(":/") == false) {
+        if (pluginId.contains(":") == false) {
             // definitely not a valid url, so assume it is a plugin name
             List<String> plugins = checkMisspelledPlugin(pluginId);
             String msg = "Unknown plugin " + pluginId;
@@ -667,7 +667,7 @@ private void installMetaPlugin(Terminal terminal, boolean isBatch, Path tmpRoot,
             pluginInfos.add(info);
             Path tmpBinDir = plugin.resolve("bin");
             if (Files.exists(tmpBinDir)) {
-                Path destBinDir = env.binFile().resolve(metaInfo.getName()).resolve(info.getName());
+                Path destBinDir = env.binFile().resolve(metaInfo.getName());
                 deleteOnFailure.add(destBinDir);
                 installBin(info, tmpBinDir, destBinDir);
             }
@@ -676,7 +676,7 @@ private void installMetaPlugin(Terminal terminal, boolean isBatch, Path tmpRoot,
             if (Files.exists(tmpConfigDir)) {
                 // some files may already exist, and we don't remove plugin config files on plugin removal,
                 // so any installed config files are left on failure too
-                Path destConfigDir = env.configFile().resolve(metaInfo.getName()).resolve(info.getName());
+                Path destConfigDir = env.configFile().resolve(metaInfo.getName());
                 installConfig(info, tmpConfigDir, destConfigDir);
             }
         }
@@ -744,7 +744,6 @@ public FileVisitResult postVisitDirectory(final Path dir, final IOException exc)
         });
     }
 
-
     /** Copies the files from {@code tmpBinDir} into {@code destBinDir}, along with permissions from dest dirs parent. */
     private void installBin(PluginInfo info, Path tmpBinDir, Path destBinDir) throws Exception {
         if (Files.isDirectory(tmpBinDir) == false) {

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -22,6 +22,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.ExitCodes;
@@ -44,6 +45,7 @@
 import org.junit.Before;
 
 import java.io.BufferedReader;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringReader;
@@ -266,18 +268,17 @@ MockTerminal installPlugin(String pluginUrl, Path home, InstallPluginCommand com
     }
 
     void assertMetaPlugin(String metaPlugin, String name, Path original, Environment env) throws IOException {
-        assertPluginInternal(name, original, env,
-            env.pluginsFile().resolve(metaPlugin), env.configFile().resolve(metaPlugin), env.binFile().resolve(metaPlugin));
+        assertPluginInternal(name, env.pluginsFile().resolve(metaPlugin));
+        assertConfigAndBin(metaPlugin, original, env);
     }
 
-
     void assertPlugin(String name, Path original, Environment env) throws IOException {
-        assertPluginInternal(name, original, env,
-            env.pluginsFile(), env.configFile(), env.binFile());
+        assertPluginInternal(name, env.pluginsFile());
+        assertConfigAndBin(name, original, env);
+        assertInstallCleaned(env);
     }
 
-    void assertPluginInternal(String name, Path original, Environment env,
-                              Path pluginsFile, Path configFile, Path binFile) throws IOException {
+    void assertPluginInternal(String name, Path pluginsFile) throws IOException {
         Path got = pluginsFile.resolve(name);
         assertTrue("dir " + name + " exists", Files.exists(got));
 
@@ -294,17 +295,19 @@ void assertPluginInternal(String name, Path original, Environment env,
                     PosixFilePermission.OTHERS_READ,
                     PosixFilePermission.OTHERS_EXECUTE));
         }
-
         assertTrue("jar was copied", Files.exists(got.resolve("plugin.jar")));
         assertFalse("bin was not copied", Files.exists(got.resolve("bin")));
         assertFalse("config was not copied", Files.exists(got.resolve("config")));
+    }
+
+    void assertConfigAndBin(String name, Path original, Environment env) throws IOException {
         if (Files.exists(original.resolve("bin"))) {
-            Path binDir = binFile.resolve(name);
+            Path binDir = env.binFile().resolve(name);
             assertTrue("bin dir exists", Files.exists(binDir));
             assertTrue("bin is a dir", Files.isDirectory(binDir));
             PosixFileAttributes binAttributes = null;
             if (isPosix) {
-                binAttributes = Files.readAttributes(binFile, PosixFileAttributes.class);
+                binAttributes = Files.readAttributes(env.binFile(), PosixFileAttributes.class);
             }
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(binDir)) {
                 for (Path file : stream) {
@@ -317,7 +320,7 @@ void assertPluginInternal(String name, Path original, Environment env,
             }
         }
         if (Files.exists(original.resolve("config"))) {
-            Path configDir = configFile.resolve(name);
+            Path configDir = env.configFile().resolve(name);
             assertTrue("config dir exists", Files.exists(configDir));
             assertTrue("config is a dir", Files.isDirectory(configDir));
 
@@ -326,7 +329,7 @@ void assertPluginInternal(String name, Path original, Environment env,
 
             if (isPosix) {
                 PosixFileAttributes configAttributes =
-                        Files.getFileAttributeView(configFile, PosixFileAttributeView.class).readAttributes();
+                        Files.getFileAttributeView(env.configFile(), PosixFileAttributeView.class).readAttributes();
                 user = configAttributes.owner();
                 group = configAttributes.group();
 
@@ -351,7 +354,6 @@ void assertPluginInternal(String name, Path original, Environment env,
                 }
             }
         }
-        assertInstallCleaned(env);
     }
 
     void assertInstallCleaned(Environment env) throws IOException {
@@ -430,6 +432,16 @@ public void testMalformedUrlNotMaven() throws Exception {
         assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
     }
 
+    public void testFileNotMaven() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        String dir = randomAlphaOfLength(10) + ":" + randomAlphaOfLength(5) + "\\" + randomAlphaOfLength(5);
+        Exception e = expectThrows(Exception.class,
+            // has two colons, so it appears similar to maven coordinates
+            () -> installPlugin("file:" + dir, env.v1()));
+        assertFalse(e.getMessage(), e.getMessage().contains("maven.org"));
+        assertTrue(e.getMessage(), e.getMessage().contains(dir));
+    }
+
     public void testUnknownPlugin() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         UserException e = expectThrows(UserException.class, () -> installPlugin("foo", env.v1()));
@@ -734,7 +746,7 @@ public void testExistingConfig() throws Exception {
 
     public void testExistingMetaConfig() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path envConfigDir = env.v2().configFile().resolve("my_plugins").resolve("fake");
+        Path envConfigDir = env.v2().configFile().resolve("my_plugins");
         Files.createDirectories(envConfigDir);
         Files.write(envConfigDir.resolve("custom.yml"), "existing config".getBytes(StandardCharsets.UTF_8));
         Path metaDir = createPluginDir(temp);

docs/build.gradle

@@ -32,8 +32,8 @@ integTestCluster {
   configFile 'analysis/stemmer_override.txt'
   configFile 'userdict_ja.txt'
   configFile 'KeywordTokenizer.rbbi'
-  extraConfigFile 'hunspell/en_US/en_US.aff', '../core/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff'
-  extraConfigFile 'hunspell/en_US/en_US.dic', '../core/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic'
+  extraConfigFile 'hunspell/en_US/en_US.aff', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff'
+  extraConfigFile 'hunspell/en_US/en_US.dic', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic'
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }

docs/java-api/admin/cluster/index.asciidoc

@@ -13,3 +13,4 @@ In the rest of this guide, we will use `client.admin().cluster()`.
 
 include::health.asciidoc[]
 
+include::stored-scripts.asciidoc[]