[Backport 2.x] Added support for jdk-21 (#513)

* Backporting changes from main branch Signed-off-by: Martin Gaievski <[email protected]>
opensearch-project · Dec 6, 2023 · 63fe67f · 63fe67f
1 parent a72b394
commit 63fe67f
Show file tree

Hide file tree

Showing 60 changed files with 160 additions and 155 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -21,7 +21,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [ubuntu-latest]
 
     name: Gradle Check Linux
@@ -55,7 +55,7 @@ jobs:
   Check-neural-search-windows:
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [windows-latest]
 
     name: Gradle Check Windows
@@ -83,7 +83,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [ubuntu-latest]
 
     name: Pre-commit Linux
@@ -117,7 +117,7 @@ jobs:
   Precommit-neural-search-windows:
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [windows-latest]
 
     name: Pre-commit Windows

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,5 +21,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Infrastructure
 ### Documentation
 ### Maintenance
+- Added support for jdk-21 ([#500](https://github.com/opensearch-project/neural-search/pull/500)))
 ### Refactoring
 Deprecate the `max_token_score` field in `neural_sparse` query clause ([#478](https://github.com/opensearch-project/neural-search/pull/478))
diff --git a/build.gradle b/build.gradle
@@ -109,8 +109,8 @@ buildscript {
 
     dependencies {
         classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}"
-        classpath "com.diffplug.spotless:spotless-plugin-gradle:5.6.1"
-        classpath "io.freefair.gradle:lombok-plugin:8.0.1"
+        classpath "com.diffplug.spotless:spotless-plugin-gradle:6.23.2"
+        classpath "io.freefair.gradle:lombok-plugin:8.4"
     }
 }
 
@@ -144,7 +144,7 @@ dependencies {
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
     compileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
     api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
-    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
+    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
     // ml-common excluded reflection for runtime so we need to add it by ourselves.
     // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9
     // TODO: Remove following three lines of dependencies if ml-common include them in their jar
@@ -255,7 +255,7 @@ testClusters.integTest {
 
     // Increase heap size from default of 512mb to 1gb. When heap size is 512mb, our integ tests sporadically fail due
     // to ml-commons memory circuit breaker exception
-    jvmArgs("-Xms1g", "-Xmx1g")
+    jvmArgs("-Xms1g", "-Xmx4g")
 }
 
 // Remote Integration Tests

diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,8 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
+distributionSha256Sum=3e1af3ae886920c3ac87f7a91f816c0c7c436f276a6eefdb3da152100fef72ae
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=e111cb9948407e26351227dabce49822fb88c37ee72f1d1582a69c68af2e702f
diff --git a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java
@@ -16,10 +16,6 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.logging.log4j.util.Strings;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.util.CollectionUtils;
@@ -35,6 +31,10 @@
 import org.opensearch.ml.common.output.model.ModelTensors;
 import org.opensearch.neuralsearch.util.RetryUtil;
 
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
 /**
  * This class will act as an abstraction on the MLCommons client for accessing the ML Capabilities
  */

diff --git a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java
@@ -14,8 +14,6 @@
 import java.util.Optional;
 import java.util.function.Supplier;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
@@ -61,6 +59,8 @@
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.watcher.ResourceWatcherService;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Neural Search plugin class
  */

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java b/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java
@@ -14,16 +14,16 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
+
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
 import lombok.extern.log4j.Log4j2;
 
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TotalHits;
-
 /**
  * Class stores collection of TopDocs for each sub query from hybrid query. Collection of results is at shard level. We do store
  * list of TopDocs and list of ScoreDoc as well as total hits for the shard.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java
@@ -14,8 +14,6 @@
 import java.util.function.Supplier;
 import java.util.stream.IntStream;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.commons.lang3.StringUtils;
 import org.opensearch.env.Environment;
 import org.opensearch.index.mapper.MapperService;
@@ -26,6 +24,8 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * The abstract class for text processing use cases. Users provide a field name map and a model id.
  * During ingestion, the processor will use the corresponding model to inference the input texts,

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java
@@ -10,9 +10,6 @@
 
 import java.util.Map;
 
-import lombok.Getter;
-import lombok.Setter;
-
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.common.Nullable;
 import org.opensearch.index.query.QueryBuilder;
@@ -22,6 +19,9 @@
 import org.opensearch.search.pipeline.Processor;
 import org.opensearch.search.pipeline.SearchRequestProcessor;
 
+import lombok.Getter;
+import lombok.Setter;
+
 /**
  * Neural Search Query Request Processor, It modifies the search request with neural query clause
  * and adds model Id if not present in the search query.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java
@@ -12,9 +12,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import lombok.AllArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.action.search.QueryPhaseResultConsumer;
 import org.opensearch.action.search.SearchPhaseContext;
 import org.opensearch.action.search.SearchPhaseName;
@@ -27,6 +24,9 @@
 import org.opensearch.search.pipeline.SearchPhaseResultsProcessor;
 import org.opensearch.search.query.QuerySearchResult;
 
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Processor for score normalization and combination on post query search results. Updates query results with
  * normalized and combined scores for next phase (typically it's FETCH)

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java
@@ -14,9 +14,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import lombok.AllArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
@@ -29,6 +26,9 @@
 import org.opensearch.search.fetch.FetchSearchResult;
 import org.opensearch.search.query.QuerySearchResult;
 
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Class abstracts steps required for score normalization and combination, this includes pre-processing of incoming data
  * and post-processing of final results

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java
@@ -9,14 +9,14 @@
 import java.util.Map;
 import java.util.function.BiConsumer;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.env.Environment;
 import org.opensearch.ingest.IngestDocument;
 import org.opensearch.neuralsearch.ml.MLCommonsClientAccessor;
 import org.opensearch.neuralsearch.util.TokenWeightUtil;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * This processor is used for user input data text sparse encoding processing, model_id can be used to indicate which model user use,
  * and field_map can be used to indicate which fields needs text embedding and the corresponding keys for the sparse encoding results.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java
@@ -9,13 +9,13 @@
 import java.util.Map;
 import java.util.function.BiConsumer;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.env.Environment;
 import org.opensearch.ingest.IngestDocument;
 import org.opensearch.neuralsearch.ml.MLCommonsClientAccessor;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * This processor is used for user input data text embedding processing, model_id can be used to indicate which model user use,
  * and field_map can be used to indicate which fields needs text embedding and the corresponding keys for the text embedding results.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java
@@ -15,8 +15,6 @@
 import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.commons.lang3.StringUtils;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
@@ -30,6 +28,8 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * This processor is used for user input data text and image embedding processing, model_id can be used to indicate which model user use,
  * and field_map can be used to indicate which fields needs embedding and the corresponding keys for the embedding results.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java
@@ -14,12 +14,12 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.commons.lang3.Range;
 
 import com.google.common.math.DoubleMath;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Collection of utility methods for score combination technique classes
  */

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java
@@ -13,13 +13,13 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHits;
 import org.opensearch.neuralsearch.processor.CompoundTopDocs;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Abstracts combination of scores in query search results.
  */

diff --git a/...ain/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java b/...ain/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java
@@ -11,9 +11,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import lombok.AllArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.neuralsearch.processor.NormalizationProcessor;
 import org.opensearch.neuralsearch.processor.NormalizationProcessorWorkflow;
 import org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique;
@@ -25,6 +22,9 @@
 import org.opensearch.search.pipeline.Processor;
 import org.opensearch.search.pipeline.SearchPhaseResultsProcessor;
 
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Factory for query results normalization processor for search pipeline. Instantiates processor based on user provided input.
  */

diff --git a/...in/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java b/...in/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java
@@ -11,13 +11,13 @@
 
 import java.util.Map;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.env.Environment;
 import org.opensearch.ingest.Processor;
 import org.opensearch.neuralsearch.ml.MLCommonsClientAccessor;
 import org.opensearch.neuralsearch.processor.SparseEncodingProcessor;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Factory for sparse encoding ingest processor for ingestion pipeline. Instantiates processor based on user provided input.
  */

diff --git a/...ava/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java b/...ava/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java
@@ -15,13 +15,13 @@
 
 import java.util.Map;
 
-import lombok.AllArgsConstructor;
-
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.env.Environment;
 import org.opensearch.neuralsearch.ml.MLCommonsClientAccessor;
 import org.opensearch.neuralsearch.processor.TextImageEmbeddingProcessor;
 
+import lombok.AllArgsConstructor;
+
 /**
  * Factory for text_image embedding ingest processor for ingestion pipeline. Instantiates processor based on user provided input.
  */

diff --git a/...va/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java b/...va/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java
@@ -9,12 +9,12 @@
 import java.util.List;
 import java.util.Objects;
 
-import lombok.ToString;
-
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.opensearch.neuralsearch.processor.CompoundTopDocs;
 
+import lombok.ToString;
+
 /**
  * Abstracts normalization of scores based on L2 method
  */

diff --git a/...rg/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java b/...rg/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java
@@ -9,14 +9,14 @@
 import java.util.List;
 import java.util.Objects;
 
-import lombok.ToString;
-
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.opensearch.neuralsearch.processor.CompoundTopDocs;
 
 import com.google.common.primitives.Floats;
 
+import lombok.ToString;
+
 /**
  * Abstracts normalization of scores based on min-max method
  */