Merge branch 'master' of github.com:elastic/elasticsearch into script…

…ing/cache-metrics-collect-pr
elastic · Oct 27, 2021 · 333dcf3 · 333dcf3
2 parents d078d19 + 6e99cdc
commit 333dcf3
Show file tree

Hide file tree

Showing 10,476 changed files with 437,807 additions and 283,032 deletions.
diff --git a/.backportrc.json b/.backportrc.json
@@ -1,13 +1,16 @@
 {
   "upstream": "elastic/elasticsearch",
   "targetBranchChoices": [
+    "master",
+    "8.0",
     "7.16",
     "7.15",
     "7.14",
     "6.8"
   ],
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
+    "^v8.1.0$": "master",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   }
 }
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -46,3 +46,4 @@ BWC_VERSION:
   - "7.15.2"
   - "7.16.0"
   - "8.0.0"
+  - "8.1.0"
diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties
@@ -5,4 +5,4 @@
 # are 'java' or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA=openjdk16
-ES_RUNTIME_JAVA=java11
+ES_RUNTIME_JAVA=openjdk17
diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml
@@ -6,9 +6,4 @@
 # or 'openjdk' followed by the major release number.
 
 ES_RUNTIME_JAVA:
-  - java11
-  - openjdk16
-  - zulu11
-  - corretto11
-  - adoptopenjdk11
   - openjdk17
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -17,3 +17,6 @@
 
 # Format aggregations and related code (server and x-pack)
 d71544976608bdb53fa4d29521fb328e1033ee2f
+
+# Reformatting of whole codebase
+12ad399c488f0cc60e19b5e1b29c6d569cb4351a
diff --git a/...va/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java b/...va/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java
@@ -60,7 +60,7 @@ public void apply(Project project) {
                     formatterConfigPath = "../" + formatterConfigPath;
                 }
 
-                java.target(getTargets(project.getPath()));
+                java.target("src/**/*.java");
 
                 // Use `@formatter:off` and `@formatter:on` to toggle formatting - ONLY IF STRICTLY NECESSARY
                 java.toggleOffOn("@formatter:off", "@formatter:on");
@@ -83,259 +83,11 @@ public void apply(Project project) {
         }
     }
 
-    @SuppressWarnings("CheckStyle")
-    private Object[] getTargets(String projectPath) {
-        if (projectPath.equals(":server")) {
-            return new String[] {
-                "src/*/java/org/elasticsearch/action/admin/cluster/repositories/**/*.java",
-                "src/*/java/org/elasticsearch/action/admin/cluster/snapshots/**/*.java",
-                "src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java",
-                "src/*/java/org/elasticsearch/index/IndexMode.java",
-                "src/*/java/org/elasticsearch/index/IndexRouting.java",
-                "src/*/java/org/elasticsearch/index/TimeSeriesModeTests.java",
-                "src/*/java/org/elasticsearch/index/snapshots/**/*.java",
-                "src/*/java/org/elasticsearch/repositories/**/*.java",
-                "src/*/java/org/elasticsearch/search/aggregations/**/*.java",
-                "src/*/java/org/elasticsearch/snapshots/**/*.java" };
-        } else if (projectPath.equals(":test:framework")) {
-            return new String[] {
-                "src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java",
-            };
-        } else {
-            // Normally this isn"t necessary, but we have Java sources in
-            // non-standard places
-            return new String[] { "src/**/*.java" };
-        }
-    }
-
     // Do not add new sub-projects here!
     private static final List<String> PROJECT_PATHS_TO_EXCLUDE = List.of(
-        ":client:benchmark",
-        ":client:client-benchmark-noop-api-plugin",
-        ":client:rest",
-        ":client:rest-high-level",
-        ":client:rest-high-level:qa:ssl-enabled",
-        ":client:sniffer",
-        ":client:test",
-        ":distribution:archives:integ-test-zip",
         ":distribution:bwc:bugfix",
         ":distribution:bwc:maintenance",
         ":distribution:bwc:minor",
-        ":distribution:bwc:staged",
-        ":distribution:docker",
-        ":docs",
-        ":example-plugins:custom-settings",
-        ":example-plugins:custom-significance-heuristic",
-        ":example-plugins:custom-suggester",
-        ":example-plugins:painless-whitelist",
-        ":example-plugins:rescore",
-        ":example-plugins:rest-handler",
-        ":example-plugins:script-expert-scoring",
-        ":example-plugins:security-authorization-engine",
-        ":libs:elasticsearch-cli",
-        ":libs:elasticsearch-core",
-        ":libs:elasticsearch-dissect",
-        ":libs:elasticsearch-geo",
-        ":libs:elasticsearch-grok",
-        ":libs:elasticsearch-lz4",
-        ":libs:elasticsearch-nio",
-        ":libs:elasticsearch-plugin-classloader",
-        ":libs:elasticsearch-secure-sm",
-        ":libs:elasticsearch-ssl-config",
-        ":libs:elasticsearch-x-content",
-        ":modules:analysis-common",
-        ":modules:ingest-common",
-        ":modules:ingest-geoip",
-        ":modules:ingest-geoip:qa:file-based-update",
-        ":modules:ingest-user-agent",
-        ":modules:lang-expression",
-        ":modules:lang-mustache",
-        ":modules:lang-painless",
-        ":modules:lang-painless:spi",
-        ":modules:mapper-extras",
-        ":modules:parent-join",
-        ":modules:percolator",
-        ":modules:rank-eval",
-        ":modules:reindex",
-        ":modules:repository-url",
-        ":modules:systemd",
-        ":modules:tasks",
-        ":modules:transport-netty4",
-        ":plugins:analysis-icu",
-        ":plugins:analysis-kuromoji",
-        ":plugins:analysis-nori",
-        ":plugins:analysis-phonetic",
-        ":plugins:analysis-smartcn",
-        ":plugins:analysis-stempel",
-        ":plugins:analysis-ukrainian",
-        ":plugins:discovery-azure-classic",
-        ":plugins:discovery-ec2",
-        ":plugins:discovery-ec2:qa:amazon-ec2",
-        ":plugins:discovery-gce",
-        ":plugins:discovery-gce:qa:gce",
-        ":plugins:ingest-attachment",
-        ":plugins:mapper-annotated-text",
-        ":plugins:mapper-murmur3",
-        ":plugins:mapper-size",
-        ":plugins:repository-azure",
-        ":plugins:repository-gcs",
-        ":plugins:repository-hdfs",
-        ":plugins:repository-hdfs:hadoop-common",
-        ":plugins:repository-s3",
-        ":plugins:store-smb",
-        ":plugins:transport-nio",
-        ":qa:ccs-rolling-upgrade-remote-cluster",
-        ":qa:ccs-unavailable-clusters",
-        ":qa:die-with-dignity",
-        ":qa:evil-tests",
-        ":qa:full-cluster-restart",
-        ":qa:logging-config",
-        ":qa:mixed-cluster",
-        ":qa:multi-cluster-search",
-        ":qa:no-bootstrap-tests",
-        ":qa:remote-clusters",
-        ":qa:repository-multi-version",
-        ":qa:rolling-upgrade",
-        ":qa:smoke-test-http",
-        ":qa:smoke-test-ingest-with-all-dependencies",
-        ":qa:smoke-test-multinode",
-        ":qa:smoke-test-plugins",
-        ":qa:snapshot-based-recoveries",
-        ":qa:snapshot-based-recoveries:azure",
-        ":qa:snapshot-based-recoveries:fs",
-        ":qa:snapshot-based-recoveries:gcs",
-        ":qa:snapshot-based-recoveries:s3",
-        ":qa:verify-version-constants",
-        ":rest-api-spec",
-        ":test:fixtures:geoip-fixture",
-        ":test:fixtures:krb5kdc-fixture",
-        ":test:fixtures:old-elasticsearch",
-        ":test:logger-usage",
-        ":x-pack:docs",
-        ":x-pack:license-tools",
-        ":x-pack:plugin",
-        ":x-pack:plugin:async-search",
-        ":x-pack:plugin:async-search:qa",
-        ":x-pack:plugin:async-search:qa:security",
-        ":x-pack:plugin:autoscaling:qa:rest",
-        ":x-pack:plugin:ccr",
-        ":x-pack:plugin:ccr:qa",
-        ":x-pack:plugin:ccr:qa:downgrade-to-basic-license",
-        ":x-pack:plugin:ccr:qa:multi-cluster",
-        ":x-pack:plugin:ccr:qa:non-compliant-license",
-        ":x-pack:plugin:ccr:qa:rest",
-        ":x-pack:plugin:ccr:qa:restart",
-        ":x-pack:plugin:ccr:qa:security",
-        ":x-pack:plugin:core",
-        ":x-pack:plugin:data-streams:qa:multi-node",
-        ":x-pack:plugin:data-streams:qa:rest",
-        ":x-pack:plugin:deprecation",
-        ":x-pack:plugin:enrich:qa:common",
-        ":x-pack:plugin:enrich:qa:rest",
-        ":x-pack:plugin:enrich:qa:rest-with-advanced-security",
-        ":x-pack:plugin:enrich:qa:rest-with-security",
-        ":x-pack:plugin:eql",
-        ":x-pack:plugin:eql:qa",
-        ":x-pack:plugin:eql:qa:common",
-        ":x-pack:plugin:eql:qa:mixed-node",
-        ":x-pack:plugin:eql:qa:multi-cluster-with-security",
-        ":x-pack:plugin:eql:qa:rest",
-        ":x-pack:plugin:eql:qa:security",
-        ":x-pack:plugin:fleet:qa:rest",
-        ":x-pack:plugin:graph",
-        ":x-pack:plugin:graph:qa:with-security",
-        ":x-pack:plugin:identity-provider",
-        ":x-pack:plugin:identity-provider:qa:idp-rest-tests",
-        ":x-pack:plugin:ilm",
-        ":x-pack:plugin:ilm:qa:multi-cluster",
-        ":x-pack:plugin:ilm:qa:multi-node",
-        ":x-pack:plugin:ilm:qa:rest",
-        ":x-pack:plugin:ilm:qa:with-security",
-        ":x-pack:plugin:mapper-constant-keyword",
-        ":x-pack:plugin:mapper-flattened",
-        ":x-pack:plugin:ml",
-        ":x-pack:plugin:ml:qa:basic-multi-node",
-        ":x-pack:plugin:ml:qa:disabled",
-        ":x-pack:plugin:ml:qa:ml-with-security",
-        ":x-pack:plugin:ml:qa:native-multi-node-tests",
-        ":x-pack:plugin:ml:qa:no-bootstrap-tests",
-        ":x-pack:plugin:ml:qa:single-node-tests",
-        ":x-pack:plugin:monitoring",
-        ":x-pack:plugin:ql",
-        ":x-pack:plugin:repository-encrypted:qa:azure",
-        ":x-pack:plugin:repository-encrypted:qa:gcs",
-        ":x-pack:plugin:repository-encrypted:qa:s3",
-        ":x-pack:plugin:rollup:qa:rest",
-        ":x-pack:plugin:search-business-rules",
-        ":x-pack:plugin:searchable-snapshots:qa:rest",
-        ":x-pack:plugin:security",
-        ":x-pack:plugin:security:cli",
-        ":x-pack:plugin:security:qa:basic-enable-security",
-        ":x-pack:plugin:security:qa:security-basic",
-        ":x-pack:plugin:security:qa:security-disabled",
-        ":x-pack:plugin:security:qa:security-not-enabled",
-        ":x-pack:plugin:security:qa:security-trial",
-        ":x-pack:plugin:security:qa:service-account",
-        ":x-pack:plugin:security:qa:smoke-test-all-realms",
-        ":x-pack:plugin:security:qa:tls-basic",
-        ":x-pack:plugin:shutdown:qa:multi-node",
-        ":x-pack:plugin:snapshot-repo-test-kit:qa:rest",
-        ":x-pack:plugin:spatial",
-        ":x-pack:plugin:sql",
-        ":x-pack:plugin:sql:jdbc",
-        ":x-pack:plugin:sql:qa",
-        ":x-pack:plugin:sql:qa:jdbc",
-        ":x-pack:plugin:sql:qa:jdbc:security",
-        ":x-pack:plugin:sql:qa:mixed-node",
-        ":x-pack:plugin:sql:qa:security",
-        ":x-pack:plugin:sql:qa:server:multi-node",
-        ":x-pack:plugin:sql:qa:server:single-node",
-        ":x-pack:plugin:sql:sql-action",
-        ":x-pack:plugin:sql:sql-cli",
-        ":x-pack:plugin:sql:sql-client",
-        ":x-pack:plugin:sql:sql-proto",
-        ":x-pack:plugin:stack:qa:rest",
-        ":x-pack:plugin:text-structure:qa:text-structure-with-security",
-        ":x-pack:plugin:transform",
-        ":x-pack:plugin:transform:qa:multi-cluster-tests-with-security",
-        ":x-pack:plugin:transform:qa:multi-node-tests",
-        ":x-pack:plugin:transform:qa:single-node-tests",
-        ":x-pack:plugin:vector-tile:qa:multi-cluster",
-        ":x-pack:plugin:vectors",
-        ":x-pack:plugin:watcher",
-        ":x-pack:plugin:watcher:qa:rest",
-        ":x-pack:plugin:watcher:qa:with-monitoring",
-        ":x-pack:plugin:watcher:qa:with-security",
-        ":x-pack:plugin:wildcard",
-        ":x-pack:qa",
-        ":x-pack:qa:core-rest-tests-with-security",
-        ":x-pack:qa:evil-tests",
-        ":x-pack:qa:full-cluster-restart",
-        ":x-pack:qa:kerberos-tests",
-        ":x-pack:qa:mixed-tier-cluster",
-        ":x-pack:qa:multi-cluster-search-security",
-        ":x-pack:qa:multi-node",
-        ":x-pack:qa:oidc-op-tests",
-        ":x-pack:qa:openldap-tests",
-        ":x-pack:qa:password-protected-keystore",
-        ":x-pack:qa:reindex-tests-with-security",
-        ":x-pack:qa:rolling-upgrade",
-        ":x-pack:qa:rolling-upgrade-multi-cluster",
-        ":x-pack:qa:runtime-fields:core-with-mapped",
-        ":x-pack:qa:runtime-fields:core-with-search",
-        ":x-pack:qa:runtime-fields:with-security",
-        ":x-pack:qa:saml-idp-tests",
-        ":x-pack:qa:security-example-spi-extension",
-        ":x-pack:qa:security-setup-password-tests",
-        ":x-pack:qa:security-tools-tests",
-        ":x-pack:qa:smoke-test-plugins",
-        ":x-pack:qa:smoke-test-plugins-ssl",
-        ":x-pack:qa:smoke-test-security-with-mustache",
-        ":x-pack:qa:third-party:active-directory",
-        ":x-pack:qa:third-party:jira",
-        ":x-pack:qa:third-party:pagerduty",
-        ":x-pack:qa:third-party:slack",
-        ":x-pack:test:idp-fixture",
-        ":x-pack:test:smb-fixture"
+        ":distribution:bwc:staged"
     );
 }
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle
@@ -33,4 +33,12 @@ configure(allprojects) {
             test.executable = rootProject.jdks.provisioned_runtime.getBinJavaPath()
         }
     }
+    project.plugins.withId("elasticsearch.testclusters") { testClustersPlugin ->
+        project.plugins.withId("elasticsearch.internal-testclusters") { internalPlugin ->
+            if (BuildParams.getIsRuntimeJavaHomeSet() == false) {
+                // If no runtime java home is set, use the bundled JDK for test clusters
+                testClustersPlugin.setRuntimeJava(providers.provider(() -> file("${rootProject.jdks.provisioned_runtime.javaHomePath}")))
+            }
+        }
+    }
 }
diff --git a/...-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/...-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
@@ -86,15 +86,23 @@ public void apply(Project project) {
         JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion"));
 
         File runtimeJavaHome = findRuntimeJavaHome();
+        boolean isRuntimeJavaHomeSet = Jvm.current().getJavaHome().equals(runtimeJavaHome) == false;
 
         File rootDir = project.getRootDir();
         GitInfo gitInfo = GitInfo.gitInfo(rootDir);
 
         BuildParams.init(params -> {
             params.reset();
             params.setRuntimeJavaHome(runtimeJavaHome);
-            params.setRuntimeJavaVersion(determineJavaVersion("runtime java.home", runtimeJavaHome, minimumRuntimeVersion));
-            params.setIsRuntimeJavaHomeSet(Jvm.current().getJavaHome().equals(runtimeJavaHome) == false);
+            // TODO: Temporarily hard-code this to 17 until we upgrade to Gradle 7.3 and bump minimumRuntimeVersion
+            params.setRuntimeJavaVersion(
+                determineJavaVersion(
+                    "runtime java.home",
+                    runtimeJavaHome,
+                    isRuntimeJavaHomeSet ? JavaVersion.VERSION_17 : Jvm.current().getJavaVersion()
+                )
+            );
+            params.setIsRuntimeJavaHomeSet(isRuntimeJavaHomeSet);
             JvmInstallationMetadata runtimeJdkMetaData = metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome).getLocation());
             params.setRuntimeJavaDetails(formatJavaVendorDetails(runtimeJdkMetaData));
             params.setJavaVersions(getAvailableJavaVersions());

diff --git a/...tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/...tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
@@ -69,8 +69,8 @@
  * This class defines gradle tasks for testing our various distribution artifacts.
  */
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.2+9";
-    private static final String SYSTEM_JDK_VENDOR = "openjdk";
+    private static final String SYSTEM_JDK_VERSION = "17+35";
+    private static final String SYSTEM_JDK_VENDOR = "adoptium";
     private static final String GRADLE_JDK_VERSION = "16.0.2+7";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -1,4 +1,4 @@
-elasticsearch     = 8.0.0
+elasticsearch     = 8.1.0
 lucene            = 9.0.0-snapshot-2719cf6630e
 
 bundled_jdk_vendor = adoptium
-Original file line number
+Diff line change
@@ Expand Up / @@ -46,3 +46,4 @@ BWC_VERSION: @@
       - "7.15.2"
       - "7.16.0"
       - "8.0.0"
+      - "8.1.0"