Merge branch 'main' into feature/extensions

Signed-off-by: Sarat Vemulapalli <[email protected]>

.ci/bwcVersions

@@ -51,3 +51,4 @@ BWC_VERSION:
   - "2.2.1"
   - "2.2.2"
   - "2.3.0"
+  - "2.4.0"

CHANGELOG.md

@@ -3,6 +3,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 ### Added
+- Add support for s390x architecture ([#4001](https://github.com/opensearch-project/OpenSearch/pull/4001))
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
 - Point in time rest layer changes for create and delete PIT API ([#4064](https://github.com/opensearch-project/OpenSearch/pull/4064))
 - Added @dreamer-89 as an Opensearch maintainer ([#4342](https://github.com/opensearch-project/OpenSearch/pull/4342))
@@ -12,17 +13,26 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
 - BWC version 2.2.2 ([#4383](https://github.com/opensearch-project/OpenSearch/pull/4383))
 - Support for labels on version bump PRs, skip label support for changelog verifier ([#4391](https://github.com/opensearch-project/OpenSearch/pull/4391))
+- Update previous release bwc version to 2.4.0 ([#4455](https://github.com/opensearch-project/OpenSearch/pull/4455))
+- 2.3.0 release notes ([#4457](https://github.com/opensearch-project/OpenSearch/pull/4457))
+
+### Dependencies
+- Bumps `org.gradle.test-retry` from 1.4.0 to 1.4.1
+- Bumps `reactor-netty-core` from 1.0.19 to 1.0.22
 
 ### Dependencies
 - Bumps `com.diffplug.spotless` from 6.9.1 to 6.10.0
 - Bumps `xmlbeans` from 5.1.0 to 5.1.1
+- Bumps azure-core-http-netty from 1.12.0 to 1.12.4([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps azure-core from 1.27.0 to 1.31.0([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
+- Bumps azure-storage-common from 12.16.0 to 12.18.0([#4160](https://github.com/opensearch-project/OpenSearch/pull/4160))
 
 ### Changed
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
 - Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
 - Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156))
-- Update to Netty 4.1.80.Final ([#4359](https://github.com/opensearch-project/OpenSearch/pull/4359))
 - Add index specific setting for remote repository ([#4253](https://github.com/opensearch-project/OpenSearch/pull/4253))
+- [Segment Replication] Update replicas to commit SegmentInfos instead of relying on SIS files from primary shards. ([#4402](https://github.com/opensearch-project/OpenSearch/pull/4402))
 
 ### Deprecated
 
@@ -42,8 +52,12 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fix flaky random test `NRTReplicationEngineTests.testUpdateSegments` ([#4352](https://github.com/opensearch-project/OpenSearch/pull/4352))
 - [Segment Replication] Extend FileChunkWriter to allow cancel on transport client ([#4386](https://github.com/opensearch-project/OpenSearch/pull/4386))
 - [Segment Replication] Add check to cancel ongoing replication with old primary on onNewCheckpoint on replica ([#4363](https://github.com/opensearch-project/OpenSearch/pull/4363))
+- Fix NoSuchFileExceptions with segment replication when computing primary metadata snapshots ([#4366](https://github.com/opensearch-project/OpenSearch/pull/4366))
 - [Segment Replication] Update flaky testOnNewCheckpointFromNewPrimaryCancelOngoingReplication unit test ([#4414](https://github.com/opensearch-project/OpenSearch/pull/4414))
 - Fixed the `_cat/shards/10_basic.yml` test cases fix.
+- [Segment Replication] Fix timeout issue by calculating time needed to process getSegmentFiles ([#4426](https://github.com/opensearch-project/OpenSearch/pull/4426))
+- [Bug]: gradle check failing with java heap OutOfMemoryError (([#4328](https://github.com/opensearch-project/OpenSearch/
+- `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362))
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))

build.gradle

@@ -56,7 +56,7 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.10.0" apply false
-  id "org.gradle.test-retry" version "1.4.0" apply false
+  id "org.gradle.test-retry" version "1.4.1" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }
@@ -264,6 +264,12 @@ tasks.register("branchConsistency") {
 allprojects {
   // configure compiler options
   tasks.withType(JavaCompile).configureEach { JavaCompile compile ->
+    options.fork = true
+
+    configure(options.forkOptions) {
+      memoryMaximumSize = project.property('options.forkOptions.memoryMaximumSize')
+    }
+
     // See please https://bugs.openjdk.java.net/browse/JDK-8209058
     if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) {
       compile.options.compilerArgs << '-Werror'
@@ -389,6 +395,10 @@ allprojects {
 // the dependency is added.
 gradle.projectsEvaluated {
   allprojects {
+    project.tasks.withType(JavaForkOptions) {
+      maxHeapSize project.property('options.forkOptions.memoryMaximumSize')
+    }
+
     if (project.path == ':test:framework') {
       // :test:framework:test cannot run before and after :server:test
       return

buildSrc/src/main/java/org/opensearch/gradle/Architecture.java

@@ -35,7 +35,8 @@
 public enum Architecture {
 
     X64,
-    ARM64;
+    ARM64,
+    S390X;
 
     public static Architecture current() {
         final String architecture = System.getProperty("os.arch", "");
@@ -45,6 +46,8 @@ public static Architecture current() {
                 return X64;
             case "aarch64":
                 return ARM64;
+            case "s390x":
+                return S390X;
             default:
                 throw new IllegalArgumentException("can not determine architecture from [" + architecture + "]");
         }

buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java

@@ -247,6 +247,9 @@ private String dependencyNotation(OpenSearchDistribution distribution) {
                     case X64:
                         classifier = ":" + distribution.getPlatform() + "-x64";
                         break;
+                    case S390X:
+                        classifier = ":" + distribution.getPlatform() + "-s390x";
+                        break;
                     default:
                         throw new IllegalArgumentException("Unsupported architecture: " + distribution.getArchitecture());
                 }

buildSrc/src/main/java/org/opensearch/gradle/Jdk.java

@@ -48,7 +48,7 @@
 
 public class Jdk implements Buildable, Iterable<File> {
 
-    private static final List<String> ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64"));
+    private static final List<String> ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64", "s390x"));
     private static final List<String> ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk"));
     private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(
         Arrays.asList("darwin", "freebsd", "linux", "mac", "windows")

buildSrc/src/test/java/org/opensearch/gradle/ArchitectureTests.java

@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle;
+
+import org.opensearch.gradle.test.GradleUnitTestCase;
+
+public class ArchitectureTests extends GradleUnitTestCase {
+
+    final String architecture = System.getProperty("os.arch", "");
+
+    public void testCurrentArchitecture() {
+        assertEquals(Architecture.X64, currentArchitecture("amd64"));
+        assertEquals(Architecture.X64, currentArchitecture("x86_64"));
+        assertEquals(Architecture.ARM64, currentArchitecture("aarch64"));
+        assertEquals(Architecture.S390X, currentArchitecture("s390x"));
+    }
+
+    public void testInvalidCurrentArchitecture() {
+        assertThrows("can not determine architecture from [", IllegalArgumentException.class, () -> currentArchitecture("fooBar64"));
+    }
+
+    /**
+     * Determines the return value of {@link Architecture#current()} based on a string representing a potential OS Architecture.
+     *
+     * @param osArchToTest  An expected value of the {@code os.arch} system property on another architecture.
+     * @return the value of the {@link Architecture} enum which would have resulted with the given value.
+     * @throws IllegalArgumentException if the string is not mapped to a value of the {@link Architecture} enum.
+     */
+    private Architecture currentArchitecture(String osArchToTest) throws IllegalArgumentException {
+        // Test new architecture
+        System.setProperty("os.arch", osArchToTest);
+        try {
+            return Architecture.current();
+        } finally {
+            // Restore actual architecture property value
+            System.setProperty("os.arch", this.architecture);
+        }
+    }
+}

buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java

@@ -108,7 +108,7 @@ public void testUnknownArchitecture() {
             "11.0.2+33",
             "linux",
             "unknown",
-            "unknown architecture [unknown] for jdk [testjdk], must be one of [aarch64, x64]"
+            "unknown architecture [unknown] for jdk [testjdk], must be one of [aarch64, x64, s390x]"
         );
     }
 

buildSrc/version.properties

@@ -21,7 +21,7 @@ asm               = 9.3
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.80.Final
+netty             = 4.1.79.Final
 joda              = 2.10.13
 
 # client dependencies

distribution/archives/build.gradle

@@ -151,6 +151,13 @@ distribution_archives {
     }
   }
 
+  linuxS390xTar {
+    archiveClassifier = 'linux-s390x'
+    content {
+      archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', false)
+    }
+  }
+
   windowsZip {
     archiveClassifier = 'windows-x64'
     content {

distribution/build.gradle

@@ -280,7 +280,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
   // Setup all required JDKs
   project.jdks {
     ['darwin', 'linux', 'windows'].each { platform ->
-      (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64'] : ['x64']).each { architecture ->
+      (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x'] : ['x64']).each { architecture ->
         "bundled_${platform}_${architecture}" {
           it.platform = platform
           it.version = VersionProperties.getBundledJdk(platform)
@@ -353,7 +353,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           }
         }
         def buildModules = buildModulesTaskProvider
-        List excludePlatforms = ['darwin-x64', 'freebsd-x64', 'linux-x64', 'linux-arm64', 'windows-x64', 'darwin-arm64']
+        List excludePlatforms = ['darwin-x64', 'freebsd-x64', 'linux-x64', 'linux-arm64', 'linux-s390x', 'windows-x64', 'darwin-arm64']
         if (platform != null) {
           excludePlatforms.remove(excludePlatforms.indexOf(platform))
         } else {

distribution/docker/build.gradle

@@ -27,11 +27,13 @@ testFixtures.useFixture()
 
 configurations {
   arm64DockerSource
+  s390xDockerSource
   dockerSource
 }
 
 dependencies {
   arm64DockerSource project(path: ":distribution:archives:linux-arm64-tar", configuration:"default")
+  s390xDockerSource project(path: ":distribution:archives:linux-s390x-tar", configuration:"default")
   dockerSource project(path: ":distribution:archives:linux-tar", configuration:"default")
 }
 
@@ -42,6 +44,8 @@ ext.expansions = { Architecture architecture, DockerBase base, boolean local ->
       classifier = "linux-arm64"
     } else if (architecture == Architecture.X64) {
       classifier = "linux-x64"
+    } else if (architecture == Architecture.S390X) {
+      classifier = "linux-s390x"
     } else {
       throw new IllegalArgumentException("Unsupported architecture [" + architecture + "]")
     }
@@ -85,12 +89,14 @@ RUN curl --retry 8 -S -L \\
 private static String buildPath(Architecture architecture, DockerBase base) {
   return 'build/' +
     (architecture == Architecture.ARM64 ? 'arm64-' : '') +
+    (architecture == Architecture.S390X ? 's390x-' : '') +
     'docker'
 }
 
 private static String taskName(String prefix, Architecture architecture, DockerBase base, String suffix) {
   return prefix +
     (architecture == Architecture.ARM64 ? 'Arm64' : '') +
+    (architecture == Architecture.S390X ? 'S390x' : '') +
     suffix
 }
 
@@ -127,6 +133,8 @@ void addCopyDockerContextTask(Architecture architecture, DockerBase base) {
 
     if (architecture == Architecture.ARM64) {
       from configurations.arm64DockerSource
+    } else if (architecture == Architecture.S390X) {
+      from configurations.s390xDockerSource
     } else {
       from configurations.dockerSource
     }

distribution/docker/docker-s390x-export/build.gradle

@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.

distribution/src/bin/opensearch-cli.bat

@@ -16,7 +16,7 @@ rem use a small heap size for the CLI tools, and thus the serial collector to
 rem avoid stealing many CPU cycles; a user can override by setting OPENSEARCH_JAVA_OPTS
 set OPENSEARCH_JAVA_OPTS=-Xms4m -Xmx64m -XX:+UseSerialGC %OPENSEARCH_JAVA_OPTS%
 
-%JAVA% ^
+"%JAVA%" ^
   %OPENSEARCH_JAVA_OPTS% ^
   -Dopensearch.path.home="%OPENSEARCH_HOME%" ^
   -Dopensearch.path.conf="%OPENSEARCH_PATH_CONF%" ^

distribution/src/bin/opensearch-env.bat

@@ -43,14 +43,14 @@ rem comparing to empty string makes this equivalent to bash -v check on env var
 rem and allows to effectively force use of the bundled jdk when launching OpenSearch
 rem by setting OPENSEARCH_JAVA_HOME= and JAVA_HOME= 
 if not "%OPENSEARCH_JAVA_HOME%" == "" (
-  set JAVA="%OPENSEARCH_JAVA_HOME%\bin\java.exe"
+  set "JAVA=%OPENSEARCH_JAVA_HOME%\bin\java.exe"
   set JAVA_TYPE=OPENSEARCH_JAVA_HOME 
 ) else if not "%JAVA_HOME%" == "" (
-  set JAVA="%JAVA_HOME%\bin\java.exe"
+  set "JAVA=%JAVA_HOME%\bin\java.exe"
   set JAVA_TYPE=JAVA_HOME
 ) else (
-  set JAVA="%OPENSEARCH_HOME%\jdk\bin\java.exe"
-  set JAVA_HOME="%OPENSEARCH_HOME%\jdk"
+  set "JAVA=%OPENSEARCH_HOME%\jdk\bin\java.exe"
+  set "JAVA_HOME=%OPENSEARCH_HOME%\jdk"
   set JAVA_TYPE=bundled jdk
 )
 
@@ -73,4 +73,4 @@ if defined JAVA_OPTS (
 )
 
 rem check the Java version
-%JAVA% -cp "%OPENSEARCH_CLASSPATH%" "org.opensearch.tools.java_version_checker.JavaVersionChecker" || exit /b 1
+"%JAVA%" -cp "%OPENSEARCH_CLASSPATH%" "org.opensearch.tools.java_version_checker.JavaVersionChecker" || exit /b 1

distribution/src/bin/opensearch-service.bat

@@ -121,7 +121,7 @@ if exist "%JAVA_HOME%\bin\server\jvm.dll" (
 
 :foundJVM
 if not defined OPENSEARCH_TMPDIR (
-  for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!OPENSEARCH_CLASSPATH!" "org.opensearch.tools.launchers.TempDirectory"`) do set OPENSEARCH_TMPDIR=%%a
+  for /f "tokens=* usebackq" %%a in (`CALL "%JAVA%" -cp "!OPENSEARCH_CLASSPATH!" "org.opensearch.tools.launchers.TempDirectory"`) do set OPENSEARCH_TMPDIR=%%a
 )
 
 rem The JVM options parser produces the final JVM options to start
@@ -135,7 +135,7 @@ rem   - third, JVM options from OPENSEARCH_JAVA_OPTS are applied
 rem   - fourth, ergonomic JVM options are applied
 
 @setlocal
-for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!OPENSEARCH_CLASSPATH!" "org.opensearch.tools.launchers.JvmOptionsParser" "!OPENSEARCH_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set OPENSEARCH_JAVA_OPTS=%%a
+for /F "usebackq delims=" %%a in (`CALL "%JAVA%" -cp "!OPENSEARCH_CLASSPATH!" "org.opensearch.tools.launchers.JvmOptionsParser" "!OPENSEARCH_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set OPENSEARCH_JAVA_OPTS=%%a
 @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%OPENSEARCH_JAVA_OPTS%" & set OPENSEARCH_JAVA_OPTS=%OPENSEARCH_JAVA_OPTS%
 
 if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (

distribution/src/bin/opensearch.bat

@@ -75,7 +75,7 @@ IF "%checkpassword%"=="Y" (
 )
 
 if not defined OPENSEARCH_TMPDIR (
-  for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!OPENSEARCH_CLASSPATH!" "org.opensearch.tools.launchers.TempDirectory"`) do set  OPENSEARCH_TMPDIR=%%a
+  for /f "tokens=* usebackq" %%a in (`CALL "%JAVA%" -cp "!OPENSEARCH_CLASSPATH!" "org.opensearch.tools.launchers.TempDirectory"`) do set  OPENSEARCH_TMPDIR=%%a
 )
 
 rem The JVM options parser produces the final JVM options to start
@@ -88,7 +88,7 @@ rem     jvm.options.d/*.options
 rem   - third, JVM options from OPENSEARCH_JAVA_OPTS are applied
 rem   - fourth, ergonomic JVM options are applied
 @setlocal
-for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!OPENSEARCH_CLASSPATH!" "org.opensearch.tools.launchers.JvmOptionsParser" "!OPENSEARCH_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set OPENSEARCH_JAVA_OPTS=%%a
+for /F "usebackq delims=" %%a in (`CALL "%JAVA%" -cp "!OPENSEARCH_CLASSPATH!" "org.opensearch.tools.launchers.JvmOptionsParser" "!OPENSEARCH_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set OPENSEARCH_JAVA_OPTS=%%a
 @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%OPENSEARCH_JAVA_OPTS%" & set OPENSEARCH_JAVA_OPTS=%OPENSEARCH_JAVA_OPTS%
 
 if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
@@ -103,7 +103,7 @@ SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^<=^^^<!
 SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^>=^^^>!
 SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^\=^^^\!
 
-ECHO.!KEYSTORE_PASSWORD!| %JAVA% %OPENSEARCH_JAVA_OPTS% -Dopensearch ^
+ECHO.!KEYSTORE_PASSWORD!| "%JAVA%" %OPENSEARCH_JAVA_OPTS% -Dopensearch ^
   -Dopensearch.path.home="%OPENSEARCH_HOME%" -Dopensearch.path.conf="%OPENSEARCH_PATH_CONF%" ^
   -Dopensearch.distribution.type="%OPENSEARCH_DISTRIBUTION_TYPE%" ^
   -Dopensearch.bundled_jdk="%OPENSEARCH_BUNDLED_JDK%" ^

modules/transport-netty4/build.gradle

@@ -144,14 +144,6 @@ thirdPartyAudit {
     'org.apache.log4j.Level',
     'org.apache.log4j.Logger',
 
-    // from io.netty.handler.ssl.OpenSslEngine (netty)
-    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
-    'org.bouncycastle.openssl.PEMParser',
-    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
-    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
-    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
-    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
-
     // from io.netty.handler.ssl.OpenSslEngine (netty)
     'io.netty.internal.tcnative.Buffer',
     'io.netty.internal.tcnative.CertificateCompressionAlgo',

modules/transport-netty4/licenses/netty-buffer-4.1.79.Final.jar.sha1

@@ -0,0 +1 @@
+6c014412b599489b1db27c6bc08d8a46da94e397

modules/transport-netty4/licenses/netty-codec-4.1.79.Final.jar.sha1

@@ -0,0 +1 @@
+18f5b02af7ca611978bc28f2cb58cbb3b9b0f0ef

modules/transport-netty4/licenses/netty-codec-http-4.1.79.Final.jar.sha1

@@ -0,0 +1 @@
+882c70bc0a30a98bf3ce477f043e967ac026044c

modules/transport-netty4/licenses/netty-codec-http2-4.1.79.Final.jar.sha1

@@ -0,0 +1 @@
+0eeffab0cd5efb699d5e4ab9b694d32fef6694b3