Merge branch 'main' into manager-timeout-index-api

Signed-off-by: Tianli Feng <[email protected]>

.ci/bwcVersions

@@ -40,3 +40,4 @@ BWC_VERSION:
   - "1.3.1"
   - "1.3.2"
   - "2.0.0"
+  - "2.1.0"

buildSrc/src/main/java/org/opensearch/gradle/Version.java

@@ -77,7 +77,7 @@ public Version(int major, int minor, int revision) {
         // currently snapshot is not taken into account
         int id = major * 10000000 + minor * 100000 + revision * 1000;
         // identify if new OpenSearch version 1
-        this.id = major == 1 || major == 2 ? id ^ MASK : id;
+        this.id = major == 1 || major == 2 || major == 3 ? id ^ MASK : id;
     }
 
     private static int parseSuffixNumber(String substring) {

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -237,6 +237,7 @@ public void runThirdPartyAudit() throws IOException {
         Set<String> jdkJarHellClasses = runJdkJarHellCheck();
 
         if (missingClassExcludes != null) {
+            assertNoPointlessExclusions("are not missing", missingClassExcludes, missingClasses);
             long bogousExcludesCount = Stream.concat(missingClassExcludes.stream(), violationsExcludes.stream())
                 .filter(each -> missingClasses.contains(each) == false)
                 .filter(each -> violationsClasses.contains(each) == false)
@@ -247,7 +248,6 @@ public void runThirdPartyAudit() throws IOException {
                     "All excluded classes seem to have no issues. " + "This is sometimes an indication that the check silently failed"
                 );
             }
-            assertNoPointlessExclusions("are not missing", missingClassExcludes, missingClasses);
             missingClasses.removeAll(missingClassExcludes);
         }
         assertNoPointlessExclusions("have no violations", violationsExcludes, violationsClasses);

buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java

@@ -53,11 +53,11 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase {
     private static Project packagesProject;
     private static Project bwcProject;
 
-    private static final Version BWC_MAJOR_VERSION = Version.fromString("5.0.0");
-    private static final Version BWC_MINOR_VERSION = Version.fromString("4.1.0");
-    private static final Version BWC_STAGED_VERSION = Version.fromString("4.0.0");
-    private static final Version BWC_BUGFIX_VERSION = Version.fromString("4.0.1");
-    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("3.90.1");
+    private static final Version BWC_MAJOR_VERSION = Version.fromString("6.0.0");
+    private static final Version BWC_MINOR_VERSION = Version.fromString("5.1.0");
+    private static final Version BWC_STAGED_VERSION = Version.fromString("5.0.0");
+    private static final Version BWC_BUGFIX_VERSION = Version.fromString("5.0.1");
+    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("4.90.1");
     private static final BwcVersions BWC_MINOR = new BwcVersions(
         new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)),
         BWC_MAJOR_VERSION

buildSrc/src/test/java/org/opensearch/gradle/VersionTests.java

@@ -65,7 +65,7 @@ public void testRelaxedVersionParsing() {
 
     public void testCompareWithStringVersions() {
         // 1.10.2 is now rebased to OpenSearch version; so this needs to report
-        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 3.0.0", Version.fromString("1.10.20").after("3.0.0"));
+        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 6.0.0", Version.fromString("1.10.20").after("6.0.0"));
         assertTrue(
             "7.0.0-alpha1 should be equal to 7.0.0-alpha1",
             Version.fromString("7.0.0-alpha1").equals(Version.fromString("7.0.0-alpha1"))

buildSrc/version.properties

@@ -1,4 +1,4 @@
-opensearch        = 2.1.0
+opensearch        = 3.0.0
 lucene            = 9.1.0
 
 bundled_jdk_vendor = adoptium

client/rest-high-level/src/test/java/org/opensearch/client/core/MainResponseTests.java

@@ -33,7 +33,6 @@
 package org.opensearch.client.core;
 
 import org.opensearch.Build;
-import org.opensearch.LegacyESVersion;
 import org.opensearch.Version;
 import org.opensearch.client.AbstractResponseTestCase;
 import org.opensearch.cluster.ClusterName;
@@ -53,7 +52,7 @@ protected org.opensearch.action.main.MainResponse createServerTestInstance(XCont
         ClusterName clusterName = new ClusterName(randomAlphaOfLength(10));
         String nodeName = randomAlphaOfLength(10);
         final String date = new Date(randomNonNegativeLong()).toString();
-        Version version = VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT);
         Build build = new Build(
             Build.Type.UNKNOWN,
             randomAlphaOfLength(8),

libs/grok/build.gradle

@@ -31,7 +31,7 @@
 dependencies {
   api 'org.jruby.joni:joni:2.1.41'
   // joni dependencies:
-  api 'org.jruby.jcodings:jcodings:1.0.44'
+  api 'org.jruby.jcodings:jcodings:1.0.57'
 
   testImplementation(project(":test:framework")) {
     exclude group: 'org.opensearch', module: 'opensearch-grok'
@@ -41,7 +41,3 @@ dependencies {
 tasks.named('forbiddenApisMain').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
-
-thirdPartyAudit.ignoreMissingClasses(
-  'org.jcodings.unicode.UnicodeCodeRange'
-)

libs/grok/licenses/jcodings-1.0.57.jar.sha1

@@ -0,0 +1 @@
+603a9ceac39cbf7f6f27fe18b2fded4714319b0a

modules/analysis-common/src/test/java/org/opensearch/analysis/common/ConcatenateGraphTokenFilterFactoryTests.java

@@ -13,18 +13,14 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.miscellaneous.ConcatenateGraphFilter;
 import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
-import org.opensearch.LegacyESVersion;
-import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.Environment;
 import org.opensearch.index.analysis.AnalysisTestsHelper;
 import org.opensearch.index.analysis.NamedAnalyzer;
 import org.opensearch.index.analysis.TokenFilterFactory;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.OpenSearchTokenStreamTestCase;
-import org.opensearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -62,61 +58,6 @@ public void testTokenizerCustomizedSeparator() throws IOException {
         assertTokenStreamContents(tokenFilter.create(tokenizer), new String[] { "PowerShot+Is+AweSome" });
     }
 
-    public void testOldLuceneVersionSeparator() throws IOException {
-        OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
-            Settings.builder()
-                .put(
-                    IndexMetadata.SETTING_VERSION_CREATED,
-                    VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, LegacyESVersion.V_7_5_2)
-                )
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_concatenate_graph.type", "concatenate_graph")
-                .put("index.analysis.filter.my_concatenate_graph.token_separator", "+") // this will be ignored
-                .build(),
-            new CommonAnalysisPlugin()
-        );
-
-        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_concatenate_graph");
-        String source = "PowerShot Is AweSome";
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-
-        // earlier Lucene version will only use Lucene's default separator
-        assertTokenStreamContents(
-            tokenFilter.create(tokenizer),
-            new String[] {
-                "PowerShot"
-                    + ConcatenateGraphFilter.DEFAULT_TOKEN_SEPARATOR
-                    + "Is"
-                    + ConcatenateGraphFilter.DEFAULT_TOKEN_SEPARATOR
-                    + "AweSome" }
-        );
-    }
-
-    public void testOldLuceneVersionNoSeparator() throws IOException {
-        OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
-            Settings.builder()
-                .put(
-                    IndexMetadata.SETTING_VERSION_CREATED,
-                    VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, LegacyESVersion.V_7_5_2)
-                )
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.my_concatenate_graph.type", "concatenate_graph")
-                .put("index.analysis.filter.my_concatenate_graph.token_separator", "+") // this will be ignored
-                .put("index.analysis.filter.my_concatenate_graph.preserve_separator", "false")
-                .build(),
-            new CommonAnalysisPlugin()
-        );
-
-        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_concatenate_graph");
-        String source = "PowerShot Is AweSome";
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
-
-        // earlier Lucene version will not add separator if preserve_separator is false
-        assertTokenStreamContents(tokenFilter.create(tokenizer), new String[] { "PowerShotIsAweSome" });
-    }
-
     public void testTokenizerEmptySeparator() throws IOException {
         OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
             Settings.builder()

modules/analysis-common/src/test/java/org/opensearch/analysis/common/EdgeNGramTokenizerTests.java

@@ -33,7 +33,6 @@
 package org.opensearch.analysis.common;
 
 import org.apache.lucene.analysis.Tokenizer;
-import org.opensearch.LegacyESVersion;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
@@ -68,34 +67,6 @@ private IndexAnalyzers buildAnalyzers(Version version, String tokenizer) throws
 
     public void testPreConfiguredTokenizer() throws IOException {
 
-        // Before 7.3 we return ngrams of length 1 only
-        {
-            Version version = VersionUtils.randomVersionBetween(
-                random(),
-                LegacyESVersion.fromString("7.0.0"),
-                VersionUtils.getPreviousVersion(LegacyESVersion.fromString("7.3.0"))
-            );
-            try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edge_ngram")) {
-                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
-                assertNotNull(analyzer);
-                assertAnalyzesTo(analyzer, "test", new String[] { "t" });
-            }
-        }
-
-        // Check deprecated name as well
-        {
-            Version version = VersionUtils.randomVersionBetween(
-                random(),
-                LegacyESVersion.fromString("7.0.0"),
-                VersionUtils.getPreviousVersion(LegacyESVersion.fromString("7.3.0"))
-            );
-            try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edgeNGram")) {
-                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
-                assertNotNull(analyzer);
-                assertAnalyzesTo(analyzer, "test", new String[] { "t" });
-            }
-        }
-
         // Afterwards, we return ngrams of length 1 and 2, to match the default factory settings
         {
             try (IndexAnalyzers indexAnalyzers = buildAnalyzers(Version.CURRENT, "edge_ngram")) {
@@ -109,7 +80,7 @@ public void testPreConfiguredTokenizer() throws IOException {
         {
             try (
                 IndexAnalyzers indexAnalyzers = buildAnalyzers(
-                    VersionUtils.randomVersionBetween(random(), LegacyESVersion.fromString("7.3.0"), Version.CURRENT),
+                    VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT),
                     "edgeNGram"
                 )
             ) {

modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java

@@ -37,7 +37,6 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.opensearch.LegacyESVersion;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
@@ -231,10 +230,7 @@ public void testChainedSynonymFilters() throws IOException {
     public void testShingleFilters() {
 
         Settings settings = Settings.builder()
-            .put(
-                IndexMetadata.SETTING_VERSION_CREATED,
-                VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT)
-            )
+            .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT))
             .put("path.home", createTempDir().toString())
             .put("index.analysis.filter.synonyms.type", "synonym")
             .putList("index.analysis.filter.synonyms.synonyms", "programmer, developer")
@@ -293,10 +289,7 @@ public void testPreconfiguredTokenFilters() throws IOException {
         );
 
         Settings settings = Settings.builder()
-            .put(
-                IndexMetadata.SETTING_VERSION_CREATED,
-                VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT)
-            )
+            .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT))
             .put("path.home", createTempDir().toString())
             .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
@@ -320,10 +313,7 @@ public void testPreconfiguredTokenFilters() throws IOException {
     public void testDisallowedTokenFilters() throws IOException {
 
         Settings settings = Settings.builder()
-            .put(
-                IndexMetadata.SETTING_VERSION_CREATED,
-                VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT)
-            )
+            .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT))
             .put("path.home", createTempDir().toString())
             .putList("common_words", "a", "b")
             .put("output_unigrams", "true")

modules/analysis-common/src/test/java/org/opensearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java

@@ -33,7 +33,6 @@
 
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.opensearch.LegacyESVersion;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
@@ -47,7 +46,6 @@
 import org.opensearch.indices.analysis.AnalysisModule;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.IndexSettingsModule;
-import org.opensearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -202,38 +200,7 @@ public void testIgnoreKeywords() throws IOException {
     }
 
     public void testPreconfiguredFilter() throws IOException {
-        // Before 7.3 we don't adjust offsets
-        {
-            Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
-            Settings indexSettings = Settings.builder()
-                .put(
-                    IndexMetadata.SETTING_VERSION_CREATED,
-                    VersionUtils.randomVersionBetween(
-                        random(),
-                        LegacyESVersion.V_7_0_0,
-                        VersionUtils.getPreviousVersion(LegacyESVersion.V_7_3_0)
-                    )
-                )
-                .put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
-                .putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph")
-                .build();
-            IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-
-            try (
-                IndexAnalyzers indexAnalyzers = new AnalysisModule(
-                    TestEnvironment.newEnvironment(settings),
-                    Collections.singletonList(new CommonAnalysisPlugin())
-                ).getAnalysisRegistry().build(idxSettings)
-            ) {
-
-                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
-                assertNotNull(analyzer);
-                assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 0 }, new int[] { 4, 4 });
-
-            }
-        }
 
-        // Afger 7.3 we do adjust offsets
         {
             Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
             Settings indexSettings = Settings.builder()

plugins/analysis-phonetic/src/test/java/org/opensearch/index/analysis/AnalysisPhoneticFactoryTests.java

@@ -32,7 +32,6 @@
 
 package org.opensearch.index.analysis;
 
-import org.opensearch.LegacyESVersion;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
@@ -65,10 +64,7 @@ public void testDisallowedWithSynonyms() throws IOException {
         AnalysisPhoneticPlugin plugin = new AnalysisPhoneticPlugin();
 
         Settings settings = Settings.builder()
-            .put(
-                IndexMetadata.SETTING_VERSION_CREATED,
-                VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT)
-            )
+            .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT))
             .put("path.home", createTempDir().toString())
             .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);

plugins/repository-azure/build.gradle

@@ -46,7 +46,7 @@ opensearchplugin {
 dependencies {
   api 'com.azure:azure-core:1.26.0'
   api 'com.azure:azure-storage-common:12.15.0'
-  api 'com.azure:azure-core-http-netty:1.11.8'
+  api 'com.azure:azure-core-http-netty:1.11.9'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"

plugins/repository-azure/licenses/azure-core-http-netty-1.11.9.jar.sha1

@@ -0,0 +1 @@
+1d1f34b3e60db038f3913007a2706a820383dc26

plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java

@@ -91,8 +91,7 @@ final class AzureStorageSettings {
         AZURE_CLIENT_PREFIX_KEY,
         "endpoint_suffix",
         key -> Setting.simpleString(key, Property.NodeScope),
-        () -> ACCOUNT_SETTING,
-        () -> KEY_SETTING
+        () -> ACCOUNT_SETTING
     );
 
     // The overall operation timeout

rest-api-spec/src/main/resources/rest-api-spec/test/tasks.list/10_basic.yml

@@ -1,6 +1,8 @@
 ---
 "tasks_list test":
   - skip:
+      version: "all"
+      reason: "AwaitsFix https://github.com/opensearch-project/OpenSearch/issues/2757"
       features: [arbitrary_key]
 
   - do:
@@ -32,4 +34,3 @@
 
   - is_true: tasks
   - match: { tasks.0.headers.X-Opaque-Id: "That is me" }
-