Merge branch 'main' into 4967_prevent_deletion_of_snapshot_backing_index

opensearch-project · Nov 4, 2022 · e9f450a · e9f450a
2 parents 5d30a56 + f051131
commit e9f450a
Show file tree

Hide file tree

Showing 286 changed files with 7,922 additions and 2,484 deletions.
diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
@@ -7,10 +7,6 @@ on:
       - 'dependabot/**'
   pull_request_target:
     types: [opened, synchronize, reopened]
-  workflow_run:
-    workflows: ["Gradle Precommit"]
-    types:
-      - completed
 
 jobs:
   gradle-check:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 ### Added
+- Add fix for auto expand replica validation ([#4994](https://github.com/opensearch-project/OpenSearch/pull/4994))
 - Add support for s390x architecture ([#4001](https://github.com/opensearch-project/OpenSearch/pull/4001))
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
 - Point in time rest layer changes for create and delete PIT API ([#4064](https://github.com/opensearch-project/OpenSearch/pull/4064))
@@ -30,6 +31,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Update GeoGrid base class access modifier to support extensibility ([#4572](https://github.com/opensearch-project/OpenSearch/pull/4572))
 - Add a new node role 'search' which is dedicated to provide search capability ([#4689](https://github.com/opensearch-project/OpenSearch/pull/4689))
 - Introduce experimental searchable snapshot API ([#4680](https://github.com/opensearch-project/OpenSearch/pull/4680))
+- Implement Searchable Snapshot using OnDemandBlockIndexInput phase 1 ([#4892](https://github.com/opensearch-project/OpenSearch/pull/4892))
 - Recommissioning of zone. REST layer support. ([#4624](https://github.com/opensearch-project/OpenSearch/pull/4604))
 - Make searchable snapshot indexes read-only but allow deletion ([#4764](https://github.com/opensearch-project/OpenSearch/pull/4764))
 - Added in-flight cancellation of SearchShardTask based on resource consumption ([#4565](https://github.com/opensearch-project/OpenSearch/pull/4565))
@@ -48,6 +50,17 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added resource usage trackers for in-flight cancellation of SearchShardTask ([#4805](https://github.com/opensearch-project/OpenSearch/pull/4805))
 - Renamed flaky tests ([#4912](https://github.com/opensearch-project/OpenSearch/pull/4912))
 - Update previous release bwc version to 2.5.0 ([#5003](https://github.com/opensearch-project/OpenSearch/pull/5003))
+- Use getParameterCount instead of getParameterTypes ([#4821](https://github.com/opensearch-project/OpenSearch/pull/4821))
+- Remote shard balancer support for searchable snapshots ([#4870](https://github.com/opensearch-project/OpenSearch/pull/4870))
+- Added search backpressure stats API ([#4932](https://github.com/opensearch-project/OpenSearch/pull/4932))
+- [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https://github.com/opensearch-project/OpenSearch/pull/5040))
+- Add Cluster manager task throttling framework. Cluster manager node will throttle task submission based on throttling thresholds.
+  This throttling will be at task type level. Data nodes will perform retries on these throttling exception with exponential delay. (PR: [#4986](https://github.com/opensearch-project/OpenSearch/pull/4986)) ( Issue : [#479](https://github.com/opensearch-project/OpenSearch/issues/479))
+    - Throttling Exception / New Backoff policy([#3527](https://github.com/opensearch-project/OpenSearch/pull/3527))
+    - Cluster Manager node side change([#3882](https://github.com/opensearch-project/OpenSearch/pull/3882))
+    - Data node side change([#4204](https://github.com/opensearch-project/OpenSearch/pull/4204))
+    - on-boarding of tasks([#4542](https://github.com/opensearch-project/OpenSearch/pull/4542))
+    - Integs ([4588](https://github.com/opensearch-project/OpenSearch/pull/4588))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
@@ -83,6 +96,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Bump `woodstox-core` to 6.4.0 ([#4947](https://github.com/opensearch-project/OpenSearch/pull/4947))
 - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979))
 - Upgrade jetty-http, kotlin-stdlib and snakeyaml ([#4963](https://github.com/opensearch-project/OpenSearch/pull/4963))
+- OpenJDK Update (October 2022 Patch releases) ([#4997](https://github.com/opensearch-project/OpenSearch/pull/4997))
+- Upgrade zookeeper dependency in hdfs-fixture ([#5007](https://github.com/opensearch-project/OpenSearch/pull/5007))
 
 ### Changed
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
@@ -106,12 +121,14 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Update to Apache Lucene 9.4.0 ([#4661](https://github.com/opensearch-project/OpenSearch/pull/4661))
 - Controlling discovery for decommissioned nodes ([#4590](https://github.com/opensearch-project/OpenSearch/pull/4590))
 - Backport Apache Lucene version change for 2.4.0 ([#4677](https://github.com/opensearch-project/OpenSearch/pull/4677))
+- Integ Tests for Awareness Attribute Decommissioning ([#4715](https://github.com/opensearch-project/OpenSearch/pull/4715))
 - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725))
 - Fix weighted routing metadata deserialization error on process restart ([#4691](https://github.com/opensearch-project/OpenSearch/pull/4691))
 - Refactor Base Action class javadocs to OpenSearch.API ([#4732](https://github.com/opensearch-project/OpenSearch/pull/4732))
 - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459))
 - Refactored BalancedAllocator.Balancer to LocalShardsBalancer ([#4761](https://github.com/opensearch-project/OpenSearch/pull/4761))
 - Fail weight update when decommission ongoing and fail decommission when attribute not weighed away ([#4839](https://github.com/opensearch-project/OpenSearch/pull/4839))
+- Skip SymbolicLinkPreservingTarIT when running on Windows ([#5023](https://github.com/opensearch-project/OpenSearch/pull/5023))
 ### Deprecated
 ### Removed
 - Remove deprecated code to add node name into log pattern of log4j property file ([#4568](https://github.com/opensearch-project/OpenSearch/pull/4568))
@@ -124,6 +141,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Remove deprecated serialization logic from pipeline aggs ([#4847](https://github.com/opensearch-project/OpenSearch/pull/4847))
 - Remove unused private methods ([#4926](https://github.com/opensearch-project/OpenSearch/pull/4926))
 - Revert PR 4656 to unblock Windows CI ([#4949](https://github.com/opensearch-project/OpenSearch/pull/4949))
+- Remove LegacyESVersion.V_7_8_ and V_7_9_ Constants ([#4855](https://github.com/opensearch-project/OpenSearch/pull/4855))
+- Remove LegacyESVersion.V_7_6_ and V_7_7_ Constants ([#4837](https://github.com/opensearch-project/OpenSearch/pull/4837))
+- Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018))
+- Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021))
 
 ### Fixed
 - `opensearch-service.bat start` and `opensearch-service.bat manager` failing to run ([#4289](https://github.com/opensearch-project/OpenSearch/pull/4289))
@@ -183,6 +204,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Fix for failing checkExtraction, checkLicense and checkNotice tasks for windows gradle check ([#4941](https://github.com/opensearch-project/OpenSearch/pull/4941))
 - Backport failures for merge conflicts on CHANGELOG.md file ([#4977](https://github.com/opensearch-project/OpenSearch/pull/4977))
+- Remove gradle-check dependency on precommit [#5027](https://github.com/opensearch-project/OpenSearch/pull/5027)
+- Fix version check for 2.x release for awareness attribute decommission([#5034](https://github.com/opensearch-project/OpenSearch/pull/5034))
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))

diff --git a/buildSrc/src/integTest/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTarIT.java b/buildSrc/src/integTest/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTarIT.java
@@ -35,6 +35,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.tools.ant.taskdefs.condition.Os;
 import org.opensearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.api.GradleException;
 import org.gradle.testkit.runner.GradleRunner;
@@ -52,6 +53,7 @@
 
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assume.assumeFalse;
 
 public class SymbolicLinkPreservingTarIT extends GradleIntegrationTestCase {
 
@@ -60,6 +62,7 @@ public class SymbolicLinkPreservingTarIT extends GradleIntegrationTestCase {
 
     @Before
     public void before() throws IOException {
+        assumeFalse("Skip tar tests on windows.", Os.isFamily(Os.FAMILY_WINDOWS));
         final Path realFolder = temporaryFolder.getRoot().toPath().resolve("real-folder");
         Files.createDirectory(realFolder);
         Files.createFile(realFolder.resolve("file"));

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
@@ -75,9 +75,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.16+8";
+    private static final String SYSTEM_JDK_VERSION = "11.0.17+8";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.4+8";
+    private static final String GRADLE_JDK_VERSION = "17.0.5+8";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

diff --git a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/JUnit3MethodProvider.java b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/JUnit3MethodProvider.java
@@ -59,7 +59,7 @@ public Collection<Method> getTestMethods(Class<?> suiteClass, ClassModel classMo
             if (m.getName().startsWith("test")
                 && Modifier.isPublic(m.getModifiers())
                 && !Modifier.isStatic(m.getModifiers())
-                && m.getParameterTypes().length == 0) {
+                && m.getParameterCount() == 0) {
                 result.add(m);
             }
         }

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -2,7 +2,7 @@ opensearch        = 3.0.0
 lucene            = 9.5.0-snapshot-a4ef70f
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 17.0.4+8
+bundled_jdk = 17.0.5+8
 
 
 

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java
@@ -1005,37 +1005,34 @@ private static void assertSyncMethod(Method method, String apiName, List<String>
         }
 
         assertEquals("incorrect number of exceptions for method [" + method + "]", 1, method.getExceptionTypes().length);
+        final Class<?>[] parameterTypes = method.getParameterTypes();
         // a few methods don't accept a request object as argument
         if (APIS_WITHOUT_REQUEST_OBJECT.contains(apiName)) {
-            assertEquals("incorrect number of arguments for method [" + method + "]", 1, method.getParameterTypes().length);
-            assertThat(
-                "the parameter to method [" + method + "] is the wrong type",
-                method.getParameterTypes()[0],
-                equalTo(RequestOptions.class)
-            );
+            assertEquals("incorrect number of arguments for method [" + method + "]", 1, method.getParameterCount());
+            assertThat("the parameter to method [" + method + "] is the wrong type", parameterTypes[0], equalTo(RequestOptions.class));
         } else {
-            assertEquals("incorrect number of arguments for method [" + method + "]", 2, method.getParameterTypes().length);
+            assertEquals("incorrect number of arguments for method [" + method + "]", 2, method.getParameterCount());
             // This is no longer true for all methods. Some methods can contain these 2 args backwards because of deprecation
-            if (method.getParameterTypes()[0].equals(RequestOptions.class)) {
+            if (parameterTypes[0].equals(RequestOptions.class)) {
                 assertThat(
                     "the first parameter to method [" + method + "] is the wrong type",
-                    method.getParameterTypes()[0],
+                    parameterTypes[0],
                     equalTo(RequestOptions.class)
                 );
                 assertThat(
                     "the second parameter to method [" + method + "] is the wrong type",
-                    method.getParameterTypes()[1].getSimpleName(),
+                    parameterTypes[1].getSimpleName(),
                     endsWith("Request")
                 );
             } else {
                 assertThat(
                     "the first parameter to method [" + method + "] is the wrong type",
-                    method.getParameterTypes()[0].getSimpleName(),
+                    parameterTypes[0].getSimpleName(),
                     endsWith("Request")
                 );
                 assertThat(
                     "the second parameter to method [" + method + "] is the wrong type",
-                    method.getParameterTypes()[1],
+                    parameterTypes[1],
                     equalTo(RequestOptions.class)
                 );
             }
@@ -1049,39 +1046,40 @@ private static void assertAsyncMethod(Map<String, Set<Method>> methods, Method m
         );
         assertThat("async method [" + method + "] should return Cancellable", method.getReturnType(), equalTo(Cancellable.class));
         assertEquals("async method [" + method + "] should not throw any exceptions", 0, method.getExceptionTypes().length);
+        final Class<?>[] parameterTypes = method.getParameterTypes();
         if (APIS_WITHOUT_REQUEST_OBJECT.contains(apiName.replaceAll("_async$", ""))) {
-            assertEquals(2, method.getParameterTypes().length);
-            assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class));
-            assertThat(method.getParameterTypes()[1], equalTo(ActionListener.class));
+            assertEquals(2, parameterTypes.length);
+            assertThat(parameterTypes[0], equalTo(RequestOptions.class));
+            assertThat(parameterTypes[1], equalTo(ActionListener.class));
         } else {
-            assertEquals("async method [" + method + "] has the wrong number of arguments", 3, method.getParameterTypes().length);
+            assertEquals("async method [" + method + "] has the wrong number of arguments", 3, method.getParameterCount());
             // This is no longer true for all methods. Some methods can contain these 2 args backwards because of deprecation
-            if (method.getParameterTypes()[0].equals(RequestOptions.class)) {
+            if (parameterTypes[0].equals(RequestOptions.class)) {
                 assertThat(
                     "the first parameter to async method [" + method + "] should be a request type",
-                    method.getParameterTypes()[0],
+                    parameterTypes[0],
                     equalTo(RequestOptions.class)
                 );
                 assertThat(
                     "the second parameter to async method [" + method + "] is the wrong type",
-                    method.getParameterTypes()[1].getSimpleName(),
+                    parameterTypes[1].getSimpleName(),
                     endsWith("Request")
                 );
             } else {
                 assertThat(
                     "the first parameter to async method [" + method + "] should be a request type",
-                    method.getParameterTypes()[0].getSimpleName(),
+                    parameterTypes[0].getSimpleName(),
                     endsWith("Request")
                 );
                 assertThat(
                     "the second parameter to async method [" + method + "] is the wrong type",
-                    method.getParameterTypes()[1],
+                    parameterTypes[1],
                     equalTo(RequestOptions.class)
                 );
             }
             assertThat(
                 "the third parameter to async method [" + method + "] is the wrong type",
-                method.getParameterTypes()[2],
+                parameterTypes[2],
                 equalTo(ActionListener.class)
             );
         }
@@ -1094,16 +1092,17 @@ private static void assertSubmitTaskMethod(
         ClientYamlSuiteRestSpec restSpec
     ) {
         String methodName = extractMethodName(apiName);
+        final Class<?>[] parameterTypes = method.getParameterTypes();
         assertTrue("submit task method [" + method.getName() + "] doesn't have corresponding sync method", methods.containsKey(methodName));
-        assertEquals("submit task method [" + method + "] has the wrong number of arguments", 2, method.getParameterTypes().length);
+        assertEquals("submit task method [" + method + "] has the wrong number of arguments", 2, method.getParameterCount());
         assertThat(
             "the first parameter to submit task method [" + method + "] is the wrong type",
-            method.getParameterTypes()[0].getSimpleName(),
+            parameterTypes[0].getSimpleName(),
             endsWith("Request")
         );
         assertThat(
             "the second parameter to submit task method [" + method + "] is the wrong type",
-            method.getParameterTypes()[1],
+            parameterTypes[1],
             equalTo(RequestOptions.class)
         );
 

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/MainResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/MainResponseTests.java
@@ -46,21 +46,16 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class MainResponseTests extends AbstractResponseTestCase<org.opensearch.action.main.MainResponse, MainResponse> {
+    private static String DISTRIBUTION = "opensearch";
+
     @Override
     protected org.opensearch.action.main.MainResponse createServerTestInstance(XContentType xContentType) {
         String clusterUuid = randomAlphaOfLength(10);
         ClusterName clusterName = new ClusterName(randomAlphaOfLength(10));
         String nodeName = randomAlphaOfLength(10);
         final String date = new Date(randomNonNegativeLong()).toString();
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT);
-        Build build = new Build(
-            Build.Type.UNKNOWN,
-            randomAlphaOfLength(8),
-            date,
-            randomBoolean(),
-            version.toString(),
-            version.before(Version.V_1_0_0) ? null : "opensearch"
-        );
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Build build = new Build(Build.Type.UNKNOWN, randomAlphaOfLength(8), date, randomBoolean(), version.toString(), DISTRIBUTION);
         return new org.opensearch.action.main.MainResponse(nodeName, version, clusterName, clusterUuid, build);
     }
 

diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/ValidateInputTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/ValidateInputTask.java
@@ -31,7 +31,7 @@ public void accept(final Tuple<TaskInput, Terminal> input) {
         // check if the elasticsearch version is supported
         if (taskInput.getVersion().isPresent()) {
             final Version version = taskInput.getVersion().get();
-            if (version.equals(LegacyESVersion.V_7_10_2) == false) {
+            if (version.equals(LegacyESVersion.fromId(7100299)) == false) {
                 throw new RuntimeException(
                     String.format(Locale.getDefault(), "The installed version %s of elasticsearch is not supported.", version)
                 );

diff --git a/...bution/tools/upgrade-cli/src/test/java/org/opensearch/upgrade/ValidateInputTaskTests.java b/...bution/tools/upgrade-cli/src/test/java/org/opensearch/upgrade/ValidateInputTaskTests.java
@@ -39,7 +39,7 @@ public void setTask() {
 
     public void testUnsupportedEsVersion() {
         TaskInput taskInput = new TaskInput(env);
-        taskInput.setVersion(LegacyESVersion.V_7_10_1);
+        taskInput.setVersion(LegacyESVersion.fromId(7100199));
 
         final RuntimeException e = expectThrows(RuntimeException.class, () -> task.accept(new Tuple<>(taskInput, terminal)));
 
@@ -51,7 +51,7 @@ public void testGetSummaryFields() {
         taskInput.setEsConfig(PathUtils.get("es_home"));
         taskInput.setCluster("some-cluster");
         taskInput.setNode("some-node");
-        taskInput.setVersion(LegacyESVersion.V_7_10_2);
+        taskInput.setVersion(LegacyESVersion.fromId(7100299));
         taskInput.setBaseUrl("some-url");
         taskInput.setPlugins(Arrays.asList("plugin-1", "plugin-2"));