Merge branch 'opensearch-project:main' into stats-rollup

shourya035 · Aug 8, 2023 · bbcbcb1 · bbcbcb1
2 parents 5fef8f1 + 2de1c24
commit bbcbcb1
Show file tree

Hide file tree

Showing 503 changed files with 7,922 additions and 5,672 deletions.
diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
@@ -1,19 +1,29 @@
-name: Gradle Precommit
+name: Gradle Precommit and Asssemble
 on: [pull_request]
 
 jobs:
   precommit:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, macos-latest]  # precommit on ubuntu-latest is run as a part of the gradle-check workflow
+        os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up JDK 11
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           java-version: 11
-          distribution: adopt
-      - name: Run Gradle
+          distribution: temurin
+          cache: gradle
+      - name: Run Gradle (precommit)
         run: |
           ./gradlew javadoc precommit --parallel
+      - name: Setup docker (missing on MacOS)
+        if: runner.os == 'macos'
+        run: |
+          brew install docker
+          colima start
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+      - name: Run Gradle (assemble)
+        run: |
+          ./gradlew assemble --parallel
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
+- Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -49,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792))
 - Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792))
 - [Remote Store] Restrict user override for remote store index level settings ([#8812](https://github.com/opensearch-project/OpenSearch/pull/8812))
+- Removed blocking wait in TransportGetSnapshotsAction which was exhausting generic threadpool ([#8377](https://github.com/opensearch-project/OpenSearch/pull/8377))
 
 ### Deprecated
 
@@ -82,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195)
 - Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606))
 - Disallow compression level to be set for default and best_compression index codecs ([#8737]()https://github.com/opensearch-project/OpenSearch/pull/8737)
+- Prioritize replica shard movement during shard relocation ([#8875](https://github.com/opensearch-project/OpenSearch/pull/8875))
 
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307))
@@ -110,7 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807))
 - Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801))
 - [Remove] Deprecated Fractional ByteSizeValue support #9005 ([#9005](https://github.com/opensearch-project/OpenSearch/pull/9005))
-
+- Make MultiBucketConsumerService thread safe to use across slices during search ([#9047](https://github.com/opensearch-project/OpenSearch/pull/9047))
 ### Deprecated
 
 ### Removed

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java
@@ -68,7 +68,9 @@ public abstract class DockerSupportService implements BuildService<DockerSupport
     // Defines the possible locations of the Docker CLI. These will be searched in order.
     private static String[] DOCKER_BINARIES_UNIX = { "/usr/bin/docker", "/usr/local/bin/docker" };
 
-    private static String[] DOCKER_BINARIES_WINDOWS = { System.getenv("PROGRAMFILES") + "\\Docker\\Docker\\resources\\bin\\docker.exe" };
+    private static String[] DOCKER_BINARIES_WINDOWS = {
+        System.getenv("PROGRAMFILES") + "\\Docker\\Docker\\resources\\bin\\docker.exe",
+        System.getenv("SystemRoot") + "\\System32\\docker.exe" /* Github Actions */ };
 
     private static String[] DOCKER_BINARIES = Os.isFamily(Os.FAMILY_WINDOWS) ? DOCKER_BINARIES_WINDOWS : DOCKER_BINARIES_UNIX;
 

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
@@ -76,7 +76,7 @@
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.common.lucene.uid.Versions;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.CollectionUtils;
+import org.opensearch.core.common.util.CollectionUtils;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.Strings;

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java
@@ -38,7 +38,6 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.client.TimedRequest;
 import org.opensearch.client.Validatable;
-import org.opensearch.common.Strings;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.common.settings.Settings;
@@ -135,7 +134,7 @@ public CreateIndexRequest settings(String source, MediaType mediaType) {
      * Allows to set the settings using a json builder.
      */
     public CreateIndexRequest settings(XContentBuilder builder) {
-        settings(Strings.toString(builder), builder.contentType());
+        settings(builder.toString(), builder.contentType());
         return this;
     }
 

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecyclePolicy.java b/client/rest-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecyclePolicy.java
@@ -33,9 +33,9 @@
 package org.opensearch.client.slm;
 
 import org.opensearch.common.Nullable;
-import org.opensearch.common.Strings;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.ParseField;
+import org.opensearch.core.common.Strings;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.ToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;

diff --git a/...t-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecyclePolicyMetadata.java b/...t-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecyclePolicyMetadata.java
@@ -33,9 +33,9 @@
 package org.opensearch.client.slm;
 
 import org.opensearch.common.Nullable;
-import org.opensearch.common.Strings;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.ParseField;
+import org.opensearch.core.common.Strings;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.ToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecycleStats.java b/client/rest-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecycleStats.java
@@ -32,10 +32,10 @@
 
 package org.opensearch.client.slm;
 
-import org.opensearch.common.Strings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.ParseField;
+import org.opensearch.core.common.Strings;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.ToXContentObject;

diff --git a/...st-high-level/src/main/java/org/opensearch/client/slm/SnapshotRetentionConfiguration.java b/...st-high-level/src/main/java/org/opensearch/client/slm/SnapshotRetentionConfiguration.java
@@ -33,10 +33,10 @@
 package org.opensearch.client.slm;
 
 import org.opensearch.common.Nullable;
-import org.opensearch.common.Strings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.ParseField;
+import org.opensearch.core.common.Strings;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.ToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java
@@ -42,7 +42,6 @@
 import org.opensearch.action.get.MultiGetResponse;
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.search.SearchRequest;
-import org.opensearch.common.Strings;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
@@ -423,7 +422,7 @@ private static BytesArray bytesBulkRequest(String localIndex, int id) throws IOE
 
         XContentBuilder source = jsonBuilder().startObject().field("field", randomRealisticUnicodeOfLengthBetween(1, 30)).endObject();
 
-        String request = Strings.toString(action) + "\n" + Strings.toString(source) + "\n";
+        String request = action + "\n" + source + "\n";
         return new BytesArray(request);
     }
 

diff --git a/...nt/rest-high-level/src/test/java/org/opensearch/client/ClusterRequestConvertersTests.java b/...nt/rest-high-level/src/test/java/org/opensearch/client/ClusterRequestConvertersTests.java
@@ -40,7 +40,7 @@
 import org.opensearch.client.cluster.RemoteInfoRequest;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.common.Priority;
-import org.opensearch.common.util.CollectionUtils;
+import org.opensearch.core.common.util.CollectionUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.apache.hc.client5.http.classic.methods.HttpPut;

diff --git a/...nt/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java b/...nt/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java
@@ -74,7 +74,7 @@
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.CollectionUtils;
+import org.opensearch.core.common.util.CollectionUtils;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.Strings;
 import org.opensearch.test.OpenSearchTestCase;

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
@@ -52,7 +52,6 @@
 import org.opensearch.action.search.SearchScrollRequest;
 import org.opensearch.client.core.CountRequest;
 import org.opensearch.client.core.CountResponse;
-import org.opensearch.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -769,7 +768,7 @@ public void testSearchScroll() throws Exception {
         for (int i = 0; i < 100; i++) {
             XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
             Request doc = new Request(HttpPut.METHOD_NAME, "/test/_doc/" + Integer.toString(i));
-            doc.setJsonEntity(Strings.toString(builder));
+            doc.setJsonEntity(builder.toString());
             client().performRequest(doc);
         }
         client().performRequest(new Request(HttpPost.METHOD_NAME, "/test/_refresh"));
@@ -837,7 +836,7 @@ public void testSearchWithPit() throws Exception {
         for (int i = 0; i < 100; i++) {
             XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
             Request doc = new Request(HttpPut.METHOD_NAME, "/test/_doc/" + Integer.toString(i));
-            doc.setJsonEntity(Strings.toString(builder));
+            doc.setJsonEntity(builder.toString());
             client().performRequest(doc);
         }
         client().performRequest(new Request(HttpPost.METHOD_NAME, "/test/_refresh"));

diff --git a/...est-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java b/...est-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java
@@ -74,7 +74,7 @@
 import org.opensearch.client.core.TermVectorsResponse;
 import org.opensearch.client.indices.CreateIndexRequest;
 import org.opensearch.client.indices.CreateIndexResponse;
-import org.opensearch.common.Strings;
+import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.unit.ByteSizeUnit;
@@ -298,15 +298,14 @@ public void testUpdate() throws Exception {
 
             Request request = new Request("POST", "/_scripts/increment-field");
             request.setJsonEntity(
-                Strings.toString(
-                    JsonXContent.contentBuilder()
-                        .startObject()
-                        .startObject("script")
-                        .field("lang", "painless")
-                        .field("source", "ctx._source.field += params.count")
-                        .endObject()
-                        .endObject()
-                )
+                JsonXContent.contentBuilder()
+                    .startObject()
+                    .startObject("script")
+                    .field("lang", "painless")
+                    .field("source", "ctx._source.field += params.count")
+                    .endObject()
+                    .endObject()
+                    .toString()
             );
             Response response = client().performRequest(request);
             assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());

diff --git a/...ution/archives/integ-test-zip/src/test/java/org/opensearch/test/rest/NodeRestUsageIT.java b/...ution/archives/integ-test-zip/src/test/java/org/opensearch/test/rest/NodeRestUsageIT.java
@@ -35,8 +35,8 @@
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
-import org.opensearch.common.Strings;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.Strings;
 import org.opensearch.search.aggregations.AggregationBuilders;
 import org.opensearch.search.builder.SearchSourceBuilder;
 

diff --git a/.../opensearch/common/collect/Iterators.java → .../opensearch/common/collect/Iterators.java b/.../opensearch/common/collect/Iterators.java → .../opensearch/common/collect/Iterators.java
diff --git a/...search/common/collect/IteratorsTests.java → ...search/common/collect/IteratorsTests.java b/...search/common/collect/IteratorsTests.java → ...search/common/collect/IteratorsTests.java
@@ -83,6 +83,7 @@ public void testRandomSingleton() {
         int numberOfIterators = randomIntBetween(1, 1000);
         int singletonIndex = randomIntBetween(0, numberOfIterators - 1);
         int value = randomInt();
+        @SuppressWarnings("rawtypes")
         Iterator<Integer>[] iterators = new Iterator[numberOfIterators];
         for (int i = 0; i < numberOfIterators; i++) {
             iterators[i] = i != singletonIndex ? empty() : singletonIterator(value);
@@ -92,6 +93,7 @@ public void testRandomSingleton() {
 
     public void testRandomIterators() {
         int numberOfIterators = randomIntBetween(1, 1000);
+        @SuppressWarnings("rawtypes")
         Iterator<Integer>[] iterators = new Iterator[numberOfIterators];
         List<Integer> values = new ArrayList<>();
         for (int i = 0; i < numberOfIterators; i++) {