Enabled Core Branching strategy and made Neural Plugin as Extensible …

…Plugin. (#87) Signed-off-by: Navneet Verma <[email protected]> Signed-off-by: Navneet Verma <[email protected]>
opensearch-project · Dec 22, 2022 · 88dfa16 · 88dfa16
1 parent de551e2
commit 88dfa16
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 36 deletions.
diff --git a/build.gradle b/build.gradle
@@ -73,9 +73,7 @@ validateNebulaPom.enabled = false
 
 buildscript {
     ext {
-        // as we don't have 3.0.0, 2.4.0 version for K-NN on darwin we need to keep OpenSearch version as 2.3 for now.
-        // Github issue: https://github.com/opensearch-project/opensearch-build/issues/2662
-        opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         version_tokens = opensearch_version.tokenize('-')

diff --git a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java
@@ -29,6 +29,7 @@
 import org.opensearch.neuralsearch.transport.MLPredictAction;
 import org.opensearch.neuralsearch.transport.MLPredictTransportAction;
 import org.opensearch.plugins.ActionPlugin;
+import org.opensearch.plugins.ExtensiblePlugin;
 import org.opensearch.plugins.IngestPlugin;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.SearchPlugin;
@@ -40,7 +41,7 @@
 /**
  * Neural Search plugin class
  */
-public class NeuralSearch extends Plugin implements ActionPlugin, SearchPlugin, IngestPlugin {
+public class NeuralSearch extends Plugin implements ActionPlugin, SearchPlugin, IngestPlugin, ExtensiblePlugin {
 
     private MLCommonsClientAccessor clientAccessor;
 

diff --git a/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java b/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java
@@ -7,7 +7,8 @@
 
 import java.io.IOException;
 
-import org.apache.http.util.EntityUtils;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.junit.Assert;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
@@ -16,7 +17,7 @@
 public class NeuralSearchIT extends OpenSearchSecureRestTestCase {
     private static final String NEURAL_SEARCH_PLUGIN_NAME = "neural-search";
 
-    public void testNeuralSearchPluginInstalled() throws IOException {
+    public void testNeuralSearchPluginInstalled() throws IOException, ParseException {
         final Request request = new Request(RestRequest.Method.GET.name(), String.join("/", "_cat", "plugins"));
         final Response response = client().performRequest(request);
         assertOK(response);

diff --git a/src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java
@@ -5,22 +5,29 @@
 
 package org.opensearch.neuralsearch;
 
+import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE;
+import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL;
+
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import org.apache.http.Header;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.Timeout;
 import org.junit.After;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
@@ -97,13 +104,20 @@ private void configureHttpsClient(final RestClientBuilder builder, final Setting
                 .orElseThrow(() -> new RuntimeException("user name is missing"));
             final String password = Optional.ofNullable(System.getProperty(SYS_PROPERTY_KEY_PASSWORD))
                 .orElseThrow(() -> new RuntimeException("password is missing"));
-            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
+            final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            final AuthScope anyScope = new AuthScope(null, -1);
+            credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(userName, password.toCharArray()));
             try {
-                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
-                    // disable the certificate since our testing cluster just uses the default security configuration
-                    .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
-                    .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build());
+                final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create()
+                    .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                    .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build())
+                    .build();
+                final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create()
+                    .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE)
+                    .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL)
+                    .setTlsStrategy(tlsStrategy)
+                    .build();
+                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager);
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
@@ -114,7 +128,12 @@ private void configureHttpsClient(final RestClientBuilder builder, final Setting
             socketTimeoutString == null ? DEFAULT_SOCKET_TIMEOUT : socketTimeoutString,
             CLIENT_SOCKET_TIMEOUT
         );
-        builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis())));
+        builder.setRequestConfigCallback(conf -> {
+            Timeout timeout = Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis()));
+            conf.setConnectTimeout(timeout);
+            conf.setResponseTimeout(timeout);
+            return conf;
+        });
         if (settings.hasValue(CLIENT_PATH_PREFIX)) {
             builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
         }
@@ -131,7 +150,7 @@ protected boolean preserveIndicesUponCompletion() {
     @After
     public void deleteExternalIndices() throws IOException {
         final Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json" + "&expand_wildcards=all"));
-        final XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue());
+        final XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType());
         try (
             final XContentParser parser = xContentType.xContent()
                 .createParser(

diff --git a/src/test/java/org/opensearch/neuralsearch/common/BaseNeuralSearchIT.java b/src/test/java/org/opensearch/neuralsearch/common/BaseNeuralSearchIT.java
@@ -5,7 +5,6 @@
 
 package org.opensearch.neuralsearch.common;
 
-import static org.apache.http.entity.ContentType.APPLICATION_JSON;
 import static org.opensearch.neuralsearch.common.VectorUtil.vectorAsListToArray;
 
 import java.io.IOException;
@@ -24,12 +23,13 @@
 import lombok.SneakyThrows;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.util.EntityUtils;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+import org.apache.hc.core5.http.message.BasicHeader;
 import org.opensearch.client.Request;
 import org.opensearch.client.RequestOptions;
 import org.opensearch.client.Response;
@@ -87,7 +87,7 @@ protected String uploadModel(String requestBody) throws Exception {
         return modelId;
     }
 
-    protected void loadModel(String modelId) throws IOException, InterruptedException {
+    protected void loadModel(String modelId) throws Exception {
         Response uploadResponse = makeRequest(
             client(),
             "POST",
@@ -368,7 +368,7 @@ protected float computeExpectedScore(String modelId, float[] indexVector, SpaceT
         return spaceType.getVectorSimilarityFunction().compare(queryVector, indexVector);
     }
 
-    protected Map<String, Object> getTaskQueryResponse(String taskId) throws IOException {
+    protected Map<String, Object> getTaskQueryResponse(String taskId) throws Exception {
         Response taskQueryResponse = makeRequest(
             client(),
             "GET",
@@ -453,7 +453,7 @@ protected static Response makeRequest(
     }
 
     protected static HttpEntity toHttpEntity(String jsonString) {
-        return new StringEntity(jsonString, APPLICATION_JSON);
+        return new StringEntity(jsonString, ContentType.APPLICATION_JSON);
     }
 
     @AllArgsConstructor

diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java
@@ -9,9 +9,9 @@
 import java.nio.file.Path;
 import java.util.Map;
 
-import org.apache.http.HttpHeaders;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.util.EntityUtils;
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.message.BasicHeader;
 import org.opensearch.client.Response;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentHelper;