Merge branch 'main' into txlog-transfer

.ci/bwcVersions

@@ -51,4 +51,5 @@ BWC_VERSION:
   - "2.2.1"
   - "2.2.2"
   - "2.3.0"
+  - "2.3.1"
   - "2.4.0"

CHANGELOG.md

@@ -15,25 +15,33 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Support for labels on version bump PRs, skip label support for changelog verifier ([#4391](https://github.com/opensearch-project/OpenSearch/pull/4391))
 - Update previous release bwc version to 2.4.0 ([#4455](https://github.com/opensearch-project/OpenSearch/pull/4455))
 - 2.3.0 release notes ([#4457](https://github.com/opensearch-project/OpenSearch/pull/4457))
-
+- Added missing javadocs for `:distribution:tools` modules ([#4483](https://github.com/opensearch-project/OpenSearch/pull/4483))
+- Add BWC version 2.3.1 ([#4513](https://github.com/opensearch-project/OpenSearch/pull/4513))
+- [Segment Replication] Add snapshot and restore tests for segment replication feature ([#3993](https://github.com/opensearch-project/OpenSearch/pull/3993))
+- Added missing javadocs for `:example-plugins` modules ([#4540](https://github.com/opensearch-project/OpenSearch/pull/4540))
 ### Dependencies
-- Bumps `org.gradle.test-retry` from 1.4.0 to 1.4.1
-- Bumps `reactor-netty-core` from 1.0.19 to 1.0.22
+- Bumps `reactor-core` from 3.4.18 to 3.4.23
 
 ### Dependencies
-- Bumps `com.diffplug.spotless` from 6.9.1 to 6.10.0
-- Bumps `xmlbeans` from 5.1.0 to 5.1.1
-- Bumps azure-core-http-netty from 1.12.0 to 1.12.4([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
-- Bumps azure-core from 1.27.0 to 1.31.0([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
-- Bumps azure-storage-common from 12.16.0 to 12.18.0([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354))
+- Bumps azure-core-http-netty from 1.12.0 to 1.12.4 ([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps azure-core from 1.27.0 to 1.31.0 ([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps azure-storage-common from 12.16.0 to 12.18.0 ([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps `org.gradle.test-retry` from 1.4.0 to 1.4.1 ([#4411](https://github.com/opensearch-project/OpenSearch/pull/4411))
+- Bumps `reactor-netty-core` from 1.0.19 to 1.0.22 ([#4447](https://github.com/opensearch-project/OpenSearch/pull/4447))
+- Bumps `reactive-streams` from 1.0.3 to 1.0.4 ([#4488](https://github.com/opensearch-project/OpenSearch/pull/4488))
+- Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
 
 ### Changed
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
 - Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
 - Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156))
+- Weighted round-robin scheduling policy for shard coordination traffic ([#4241](https://github.com/opensearch-project/OpenSearch/pull/4241))
 - Add index specific setting for remote repository ([#4253](https://github.com/opensearch-project/OpenSearch/pull/4253))
 - [Segment Replication] Update replicas to commit SegmentInfos instead of relying on SIS files from primary shards. ([#4402](https://github.com/opensearch-project/OpenSearch/pull/4402))
-- Introduce remote translog transfer support([#4480](https://github.com/opensearch-project/OpenSearch/pull/4480))
+- [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
+- [Remote Store] Change behaviour in replica recovery for remote translog enabled indices ([#4318](https://github.com/opensearch-project/OpenSearch/pull/4318))
+- [Remote Store] Introduce remote translog transfer support([#4480](https://github.com/opensearch-project/OpenSearch/pull/4480))
 
 ### Deprecated
 
@@ -59,6 +67,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Segment Replication] Fix timeout issue by calculating time needed to process getSegmentFiles ([#4426](https://github.com/opensearch-project/OpenSearch/pull/4426))
 - [Bug]: gradle check failing with java heap OutOfMemoryError (([#4328](https://github.com/opensearch-project/OpenSearch/
 - `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362))
+- Getting security exception due to access denied 'java.lang.RuntimePermission' 'accessDeclaredMembers' when trying to get snapshot with S3 IRSA ([#4469](https://github.com/opensearch-project/OpenSearch/pull/4469))
+- Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484))
+- Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494))
+- Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556))
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))
@@ -68,6 +80,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
 - Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348))
 - Added RestLayer Changes for PIT stats ([#4217](https://github.com/opensearch-project/OpenSearch/pull/4217))
+- Added GeoBounds aggregation on GeoShape field type.([#4266](https://github.com/opensearch-project/OpenSearch/pull/4266))
+  - Addition of Doc values on the GeoShape Field
+  - Addition of GeoShape ValueSource level code interfaces for accessing the DocValues.
+  - Addition of Missing Value feature in the GeoShape Aggregations.
 
 ### Changed
 
@@ -83,4 +99,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 
 [Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...HEAD
-[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x
+[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x

build.gradle

@@ -55,7 +55,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.10.0" apply false
+  id "com.diffplug.spotless" version "6.11.0" apply false
   id "org.gradle.test-retry" version "1.4.1" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/version.properties

@@ -9,9 +9,9 @@ bundled_jdk = 17.0.4+8
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.13.3
-jackson_databind  = 2.13.3
-snakeyaml         = 1.31
+jackson           = 2.13.4
+jackson_databind  = 2.13.4
+snakeyaml         = 1.32
 icu4j             = 70.1
 supercsv          = 2.4.0
 log4j             = 2.17.1

client/sniffer/licenses/jackson-core-2.13.4.jar.sha1

@@ -0,0 +1 @@
+0cf934c681294b97ef6d80082faeefbe1edadf56

distribution/tools/java-version-checker/src/main/java/org/opensearch/tools/java_version_checker/SuppressForbidden.java

@@ -43,5 +43,10 @@
 @Retention(RetentionPolicy.CLASS)
 @Target({ ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE })
 public @interface SuppressForbidden {
+    /**
+     * The argument to this annotation, specifying the reason a forbidden API is being used.
+     *
+     * @return The reason the error is being suppressed.
+     */
     String reason();
 }

distribution/tools/java-version-checker/src/main/java/org/opensearch/tools/java_version_checker/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Tools to validate minimum version of the runtime Java.
+ */
+package org.opensearch.tools.java_version_checker;

distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/KeyStoreCli.java

@@ -36,7 +36,7 @@
 import org.opensearch.cli.Terminal;
 
 /**
- * A cli tool for managing secrets in the opensearch keystore.
+ * A CLI tool for managing secrets in the OpenSearch keystore.
  */
 public class KeyStoreCli extends LoggingAwareMultiCommand {
 
@@ -52,6 +52,12 @@ private KeyStoreCli() {
         subcommands.put("has-passwd", new HasPasswordKeyStoreCommand());
     }
 
+    /**
+     * Main entry point for the OpenSearch Keystore CLI tool.
+     *
+     * @param args  CLI commands for managing secrets.
+     * @throws Exception if an exception was encountered executing the command.
+     */
     public static void main(String[] args) throws Exception {
         exit(new KeyStoreCli().main(args, Terminal.DEFAULT));
     }

distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Classes implementing a CLI tool for managing secrets in the OpenSearch keystore.
+ */
+package org.opensearch.common.settings;

distribution/tools/launchers/build.gradle

@@ -54,6 +54,5 @@ testingConventions {
 }
 
 javadoc.enabled = false
-missingJavadoc.enabled = false
 loggerUsageCheck.enabled = false
 jarHell.enabled = false

distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Classes implementing utility methods for launching JVMs.
+ */
+package org.opensearch.tools.launchers;

distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/PluginCli.java

@@ -42,7 +42,7 @@
 import java.util.Collections;
 
 /**
- * A cli tool for adding, removing and listing plugins for opensearch.
+ * A CLI tool for adding, removing and listing plugins for OpenSearch.
  */
 public class PluginCli extends LoggingAwareMultiCommand {
 
@@ -56,6 +56,12 @@ private PluginCli() {
         commands = Collections.unmodifiableCollection(subcommands.values());
     }
 
+    /**
+     * Main entry point for the OpenSearch Plugin CLI tool.
+     *
+     * @param args  CLI commands for managing plugins.
+     * @throws Exception if an exception was encountered executing the command.
+     */
     public static void main(String[] args) throws Exception {
         exit(new PluginCli().main(args, Terminal.DEFAULT));
     }

distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/PluginHelper.java

@@ -19,6 +19,8 @@
  */
 public class PluginHelper {
 
+    private PluginHelper() {}
+
     /**
      * Verify if a plugin exists with any folder name.
      * @param pluginPath   the path for the plugins directory.

distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Classes implementing a CLI tool for managing plugins in OpenSearch.
+ */
+package org.opensearch.plugins;

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.13.4.jar.sha1

@@ -0,0 +1 @@
+858c6cc78e1f08a885b1613e1d817c829df70a6e

distribution/tools/upgrade-cli/licenses/jackson-databind-2.13.4.jar.sha1

@@ -0,0 +1 @@
+98b0edfa8e4084078f10b7b356c300ded4a71491

gradle/missing-javadoc.gradle

@@ -95,18 +95,6 @@ configure([
   project(":client:client-benchmark-noop-api-plugin"),
   project(":client:rest-high-level"),
   project(":client:test"),
-  project(":distribution:tools:java-version-checker"),
-  project(":distribution:tools:keystore-cli"),
-  project(":distribution:tools:launchers"),
-  project(":distribution:tools:plugin-cli"),
-  project(":doc-tools"),
-  project(":example-plugins:custom-settings"),
-  project(":example-plugins:custom-significance-heuristic"),
-  project(":example-plugins:custom-suggester"),
-  project(":example-plugins:painless-allowlist"),
-  project(":example-plugins:rescore"),
-  project(":example-plugins:rest-handler"),
-  project(":example-plugins:script-expert-scoring"),
   project(":libs:opensearch-cli"),
   project(":libs:opensearch-core"),
   project(":libs:opensearch-dissect"),
@@ -159,9 +147,7 @@ configure([
   project(":plugins:store-smb"),
   project(":plugins:transport-nio"),
   project(":qa:die-with-dignity"),
-  project(":qa:os"),
   project(":qa:wildfly"),
-  project(":rest-api-spec"),
   project(":test:external-modules:test-delayed-aggs"),
   project(":test:fixtures:azure-fixture"),
   project(":test:fixtures:gcs-fixture"),

libs/geo/src/main/java/org/opensearch/geometry/GeometryCollection.java

@@ -88,6 +88,15 @@ public G get(int i) {
         return shapes.get(i);
     }
 
+    /**
+     * Returns a {@link List} of All {@link Geometry} present in this collection.
+     *
+     * @return a {@link List} of All {@link Geometry}
+     */
+    public List<G> getAll() {
+        return shapes;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

libs/x-content/licenses/jackson-core-2.13.4.jar.sha1

@@ -0,0 +1 @@
+0cf934c681294b97ef6d80082faeefbe1edadf56

libs/x-content/licenses/jackson-dataformat-cbor-2.13.4.jar.sha1

@@ -0,0 +1 @@
+ccaf21e6a02a20cae6591a12d20bf310544cf3ee

libs/x-content/licenses/jackson-dataformat-smile-2.13.4.jar.sha1

@@ -0,0 +1 @@
+4161a7c3914a12e7b7940ea53eb3c53e17aea91b

libs/x-content/licenses/jackson-dataformat-yaml-2.13.4.jar.sha1

@@ -0,0 +1 @@
+3142ec201e878372d1561e64bd1a947d9e88a03d

libs/x-content/licenses/snakeyaml-1.32.jar.sha1

@@ -0,0 +1 @@
+e80612549feb5c9191c498de628c1aa80693cf0b

modules/geo/build.gradle

@@ -40,6 +40,7 @@ restResources {
     includeCore '_common', 'indices', 'index', 'search', 'bulk'
   }
 }
+
 artifacts {
   restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test'))
 }

modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java

@@ -21,6 +21,9 @@
  * for the test cluster on which integration tests are running.
  */
 public abstract class GeoModulePluginIntegTestCase extends OpenSearchIntegTestCase {
+
+    protected static final double GEOHASH_TOLERANCE = 1E-5D;
+
     /**
      * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this
      * geo plugin is not getting packaged in a zip, we need to load it before the tests run.