Merge branch 'main' into feature/wrr-put-api

CHANGELOG.md

@@ -6,6 +6,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add support for s390x architecture ([#4001](https://github.com/opensearch-project/OpenSearch/pull/4001))
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
 - Point in time rest layer changes for create and delete PIT API ([#4064](https://github.com/opensearch-project/OpenSearch/pull/4064))
+- Point in time rest layer changes for list PIT and PIT segments API ([#4388](https://github.com/opensearch-project/OpenSearch/pull/4388))
 - Added @dreamer-89 as an Opensearch maintainer ([#4342](https://github.com/opensearch-project/OpenSearch/pull/4342))
 - Added release notes for 1.3.5 ([#4343](https://github.com/opensearch-project/OpenSearch/pull/4343))
 - Added release notes for 2.2.1 ([#4344](https://github.com/opensearch-project/OpenSearch/pull/4344))
@@ -18,20 +19,22 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added missing javadocs for `:distribution:tools` modules ([#4483](https://github.com/opensearch-project/OpenSearch/pull/4483))
 - Add BWC version 2.3.1 ([#4513](https://github.com/opensearch-project/OpenSearch/pull/4513))
 - [Segment Replication] Add snapshot and restore tests for segment replication feature ([#3993](https://github.com/opensearch-project/OpenSearch/pull/3993))
-
+- Added missing javadocs for `:example-plugins` modules ([#4540](https://github.com/opensearch-project/OpenSearch/pull/4540))
 ### Dependencies
-- Bumps `reactive-streams` from 1.0.3 to 1.0.4
+- Bumps `log4j-core` from 2.18.0 to 2.19.0
 
-### Dependencies
-- Bumps `org.gradle.test-retry` from 1.4.0 to 1.4.1
-- Bumps `reactor-netty-core` from 1.0.19 to 1.0.22
 
 ### Dependencies
-- Bumps `com.diffplug.spotless` from 6.9.1 to 6.10.0
-- Bumps `xmlbeans` from 5.1.0 to 5.1.1
-- Bumps azure-core-http-netty from 1.12.0 to 1.12.4([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
-- Bumps azure-core from 1.27.0 to 1.31.0([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
-- Bumps azure-storage-common from 12.16.0 to 12.18.0([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354))
+- Bumps `azure-core-http-netty` from 1.12.0 to 1.12.4 ([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps `azure-core` from 1.27.0 to 1.31.0 ([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps `azure-storage-common` from 12.16.0 to 12.18.0 ([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps `org.gradle.test-retry` from 1.4.0 to 1.4.1 ([#4411](https://github.com/opensearch-project/OpenSearch/pull/4411))
+- Bumps `reactor-netty-core` from 1.0.19 to 1.0.22 ([#4447](https://github.com/opensearch-project/OpenSearch/pull/4447))
+- Bumps `reactive-streams` from 1.0.3 to 1.0.4 ([#4488](https://github.com/opensearch-project/OpenSearch/pull/4488))
+- Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
+- Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
+- Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
 
 ### Changed
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
@@ -43,10 +46,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
 - [Remote Store] Change behaviour in replica recovery for remote translog enabled indices ([#4318](https://github.com/opensearch-project/OpenSearch/pull/4318))
 - PUT api for weighted shard routing ([#4272](https://github.com/opensearch-project/OpenSearch/pull/4272))
+- Unmute test RelocationIT.testRelocationWhileIndexingRandom ([#4580](https://github.com/opensearch-project/OpenSearch/pull/4580))
+- Add DecommissionService and helper to execute awareness attribute decommissioning ([#4084](https://github.com/opensearch-project/OpenSearch/pull/4084))
 
 ### Deprecated
 
 ### Removed
+- Remove deprecated code to add node name into log pattern of log4j property file ([#4568](https://github.com/opensearch-project/OpenSearch/pull/4568))
 
 ### Fixed
 - `opensearch-service.bat start` and `opensearch-service.bat manager` failing to run ([#4289](https://github.com/opensearch-project/OpenSearch/pull/4289))
@@ -71,6 +77,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Getting security exception due to access denied 'java.lang.RuntimePermission' 'accessDeclaredMembers' when trying to get snapshot with S3 IRSA ([#4469](https://github.com/opensearch-project/OpenSearch/pull/4469))
 - Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484))
 - Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494))
+- [Segment Replication] Ignore lock file when testing cleanupAndPreserveLatestCommitPoint ([#4544](https://github.com/opensearch-project/OpenSearch/pull/4544))
+- Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556))
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))
@@ -80,6 +88,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
 - Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348))
 - Added RestLayer Changes for PIT stats ([#4217](https://github.com/opensearch-project/OpenSearch/pull/4217))
+- Added GeoBounds aggregation on GeoShape field type.([#4266](https://github.com/opensearch-project/OpenSearch/pull/4266))
+  - Addition of Doc values on the GeoShape Field
+  - Addition of GeoShape ValueSource level code interfaces for accessing the DocValues.
+  - Addition of Missing Value feature in the GeoShape Aggregations.
 
 ### Changed
 

build.gradle

@@ -55,7 +55,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.10.0" apply false
+  id "com.diffplug.spotless" version "6.11.0" apply false
   id "org.gradle.test-retry" version "1.4.1" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -16,7 +16,7 @@ repositories {
   mavenCentral()
 }
 dependencies {
-  implementation 'org.apache.logging.log4j:log4j-core:2.18.0'
+  implementation 'org.apache.logging.log4j:log4j-core:2.19.0'
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

buildSrc/version.properties

@@ -9,9 +9,9 @@ bundled_jdk = 17.0.4+8
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.13.3
-jackson_databind  = 2.13.3
-snakeyaml         = 1.31
+jackson           = 2.13.4
+jackson_databind  = 2.13.4
+snakeyaml         = 1.32
 icu4j             = 70.1
 supercsv          = 2.4.0
 log4j             = 2.17.1

client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java

@@ -498,6 +498,10 @@ static Request deleteAllPits() {
         return new Request(HttpDelete.METHOD_NAME, "/_search/point_in_time/_all");
     }
 
+    static Request getAllPits() {
+        return new Request(HttpGet.METHOD_NAME, "/_search/point_in_time/_all");
+    }
+
     static Request multiSearch(MultiSearchRequest multiSearchRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_msearch");
 

client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java

@@ -63,6 +63,7 @@
 import org.opensearch.action.search.CreatePitResponse;
 import org.opensearch.action.search.DeletePitRequest;
 import org.opensearch.action.search.DeletePitResponse;
+import org.opensearch.action.search.GetAllPitNodesResponse;
 import org.opensearch.action.search.MultiSearchRequest;
 import org.opensearch.action.search.MultiSearchResponse;
 import org.opensearch.action.search.SearchRequest;
@@ -1368,6 +1369,40 @@ public final Cancellable deleteAllPitsAsync(RequestOptions options, ActionListen
         );
     }
 
+    /**
+     * Get all point in time searches using list all PITs API
+     *
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     */
+    public final GetAllPitNodesResponse getAllPits(RequestOptions options) throws IOException {
+        return performRequestAndParseEntity(
+            new MainRequest(),
+            (request) -> RequestConverters.getAllPits(),
+            options,
+            GetAllPitNodesResponse::fromXContent,
+            emptySet()
+        );
+    }
+
+    /**
+     * Asynchronously get all point in time searches using list all PITs API
+     *
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     * @return the response
+     */
+    public final Cancellable getAllPitsAsync(RequestOptions options, ActionListener<GetAllPitNodesResponse> listener) {
+        return performRequestAsyncAndParseEntity(
+            new MainRequest(),
+            (request) -> RequestConverters.getAllPits(),
+            options,
+            GetAllPitNodesResponse::fromXContent,
+            listener,
+            emptySet()
+        );
+    }
+
     /**
      * Clears one or more scroll ids using the Clear Scroll API.
      *

client/rest-high-level/src/test/java/org/opensearch/client/PitIT.java

@@ -18,12 +18,14 @@
 import org.opensearch.action.search.DeletePitInfo;
 import org.opensearch.action.search.DeletePitRequest;
 import org.opensearch.action.search.DeletePitResponse;
+import org.opensearch.action.search.GetAllPitNodesResponse;
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 /**
  * Tests point in time API with rest high level client
@@ -52,21 +54,24 @@ public void indexDocuments() throws IOException {
 
     public void testCreateAndDeletePit() throws IOException {
         CreatePitRequest pitRequest = new CreatePitRequest(new TimeValue(1, TimeUnit.DAYS), true, "index");
-        CreatePitResponse pitResponse = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
-        assertTrue(pitResponse.getId() != null);
-        assertEquals(1, pitResponse.getTotalShards());
-        assertEquals(1, pitResponse.getSuccessfulShards());
-        assertEquals(0, pitResponse.getFailedShards());
-        assertEquals(0, pitResponse.getSkippedShards());
+        CreatePitResponse createPitResponse = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
+        assertTrue(createPitResponse.getId() != null);
+        assertEquals(1, createPitResponse.getTotalShards());
+        assertEquals(1, createPitResponse.getSuccessfulShards());
+        assertEquals(0, createPitResponse.getFailedShards());
+        assertEquals(0, createPitResponse.getSkippedShards());
+        GetAllPitNodesResponse getAllPitResponse = highLevelClient().getAllPits(RequestOptions.DEFAULT);
+        List<String> pits = getAllPitResponse.getPitInfos().stream().map(r -> r.getPitId()).collect(Collectors.toList());
+        assertTrue(pits.contains(createPitResponse.getId()));
         List<String> pitIds = new ArrayList<>();
-        pitIds.add(pitResponse.getId());
+        pitIds.add(createPitResponse.getId());
         DeletePitRequest deletePitRequest = new DeletePitRequest(pitIds);
         DeletePitResponse deletePitResponse = execute(deletePitRequest, highLevelClient()::deletePit, highLevelClient()::deletePitAsync);
         assertTrue(deletePitResponse.getDeletePitResults().get(0).isSuccessful());
-        assertTrue(deletePitResponse.getDeletePitResults().get(0).getPitId().equals(pitResponse.getId()));
+        assertTrue(deletePitResponse.getDeletePitResults().get(0).getPitId().equals(createPitResponse.getId()));
     }
 
-    public void testDeleteAllPits() throws IOException {
+    public void testDeleteAllAndListAllPits() throws IOException {
         CreatePitRequest pitRequest = new CreatePitRequest(new TimeValue(1, TimeUnit.DAYS), true, "index");
         CreatePitResponse pitResponse = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
         CreatePitResponse pitResponse1 = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
@@ -80,6 +85,11 @@ public void testDeleteAllPits() throws IOException {
         pitResponse1 = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
         assertTrue(pitResponse.getId() != null);
         assertTrue(pitResponse1.getId() != null);
+        GetAllPitNodesResponse getAllPitResponse = highLevelClient().getAllPits(RequestOptions.DEFAULT);
+
+        List<String> pits = getAllPitResponse.getPitInfos().stream().map(r -> r.getPitId()).collect(Collectors.toList());
+        assertTrue(pits.contains(pitResponse.getId()));
+        assertTrue(pits.contains(pitResponse1.getId()));
         ActionListener<DeletePitResponse> deletePitListener = new ActionListener<>() {
             @Override
             public void onResponse(DeletePitResponse response) {
@@ -95,8 +105,27 @@ public void onFailure(Exception e) {
                 }
             }
         };
+        final CreatePitResponse pitResponse3 = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
+
+        ActionListener<GetAllPitNodesResponse> getPitsListener = new ActionListener<GetAllPitNodesResponse>() {
+            @Override
+            public void onResponse(GetAllPitNodesResponse response) {
+                List<String> pits = response.getPitInfos().stream().map(r -> r.getPitId()).collect(Collectors.toList());
+                assertTrue(pits.contains(pitResponse3.getId()));
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                if (!(e instanceof OpenSearchStatusException)) {
+                    throw new AssertionError("List all PITs failed", e);
+                }
+            }
+        };
+        highLevelClient().getAllPitsAsync(RequestOptions.DEFAULT, getPitsListener);
         highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
         // validate no pits case
+        getAllPitResponse = highLevelClient().getAllPits(RequestOptions.DEFAULT);
+        assertTrue(getAllPitResponse.getPitInfos().size() == 0);
         highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
     }
 }

client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java

@@ -135,6 +135,7 @@ public class RestHighLevelClientTests extends OpenSearchTestCase {
         "ping",
         "info",
         "delete_all_pits",
+        "get_all_pits",
         // security
         "security.get_ssl_certificates",
         "security.authenticate",

client/sniffer/licenses/jackson-core-2.13.4.jar.sha1

@@ -0,0 +1 @@
+0cf934c681294b97ef6d80082faeefbe1edadf56

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.13.4.jar.sha1

@@ -0,0 +1 @@
+858c6cc78e1f08a885b1613e1d817c829df70a6e

distribution/tools/upgrade-cli/licenses/jackson-databind-2.13.4.jar.sha1

@@ -0,0 +1 @@
+98b0edfa8e4084078f10b7b356c300ded4a71491

gradle/missing-javadoc.gradle

@@ -95,14 +95,6 @@ configure([
   project(":client:client-benchmark-noop-api-plugin"),
   project(":client:rest-high-level"),
   project(":client:test"),
-  project(":doc-tools"),
-  project(":example-plugins:custom-settings"),
-  project(":example-plugins:custom-significance-heuristic"),
-  project(":example-plugins:custom-suggester"),
-  project(":example-plugins:painless-allowlist"),
-  project(":example-plugins:rescore"),
-  project(":example-plugins:rest-handler"),
-  project(":example-plugins:script-expert-scoring"),
   project(":libs:opensearch-cli"),
   project(":libs:opensearch-core"),
   project(":libs:opensearch-dissect"),
@@ -155,9 +147,7 @@ configure([
   project(":plugins:store-smb"),
   project(":plugins:transport-nio"),
   project(":qa:die-with-dignity"),
-  project(":qa:os"),
   project(":qa:wildfly"),
-  project(":rest-api-spec"),
   project(":test:external-modules:test-delayed-aggs"),
   project(":test:fixtures:azure-fixture"),
   project(":test:fixtures:gcs-fixture"),

libs/geo/src/main/java/org/opensearch/geometry/GeometryCollection.java

@@ -88,6 +88,15 @@ public G get(int i) {
         return shapes.get(i);
     }
 
+    /**
+     * Returns a {@link List} of All {@link Geometry} present in this collection.
+     *
+     * @return a {@link List} of All {@link Geometry}
+     */
+    public List<G> getAll() {
+        return shapes;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

libs/x-content/licenses/jackson-core-2.13.4.jar.sha1

@@ -0,0 +1 @@
+0cf934c681294b97ef6d80082faeefbe1edadf56

libs/x-content/licenses/jackson-dataformat-cbor-2.13.4.jar.sha1

@@ -0,0 +1 @@
+ccaf21e6a02a20cae6591a12d20bf310544cf3ee

libs/x-content/licenses/jackson-dataformat-smile-2.13.4.jar.sha1

@@ -0,0 +1 @@
+4161a7c3914a12e7b7940ea53eb3c53e17aea91b

libs/x-content/licenses/jackson-dataformat-yaml-2.13.4.jar.sha1

@@ -0,0 +1 @@
+3142ec201e878372d1561e64bd1a947d9e88a03d

libs/x-content/licenses/snakeyaml-1.32.jar.sha1

@@ -0,0 +1 @@
+e80612549feb5c9191c498de628c1aa80693cf0b

modules/geo/build.gradle

@@ -40,6 +40,7 @@ restResources {
     includeCore '_common', 'indices', 'index', 'search', 'bulk'
   }
 }
+
 artifacts {
   restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test'))
 }

modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java

@@ -21,6 +21,9 @@
  * for the test cluster on which integration tests are running.
  */
 public abstract class GeoModulePluginIntegTestCase extends OpenSearchIntegTestCase {
+
+    protected static final double GEOHASH_TOLERANCE = 1E-5D;
+
     /**
      * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this
      * geo plugin is not getting packaged in a zip, we need to load it before the tests run.