Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SolrClientCache switch to Http2SolrClient #2764

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

SolrClientCache switch to Http2SolrClient #2764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a73f175
SchemaDesigner API switch to CloudSolrClient
iamsanjay Oct 15, 2024
86cf9fc
AnalyzeField: use NamedList, simpler. Don't need InputStreamParser
dsmiley Nov 28, 2024
d3a2995
Merge branch 'main' into jetty_solrClientCache
iamsanjay Dec 3, 2024
ae9c5bb
revert failing test changes
iamsanjay Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion solr/core/src/java/org/apache/solr/core/CoreContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -831,7 +831,7 @@ private void loadInternal() {
solrClientProvider =
new HttpSolrClientProvider(cfg.getUpdateShardHandlerConfig(), solrMetricsContext);
updateShardHandler.initializeMetrics(solrMetricsContext, "updateShardHandler");
solrClientCache = new SolrClientCache(updateShardHandler.getDefaultHttpClient());
solrClientCache = new SolrClientCache(solrClientProvider.getSolrClient());

Map<String, CacheConfig> cachesConfig = cfg.getCachesConfig();
if (cachesConfig.isEmpty()) {
Expand Down
2 changes: 1 addition & 1 deletion solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ public void getSampleValue(SolrQueryRequest req, SolrQueryResponse rsp) throws I
}

if (textValue != null) {
Map<String, Object> analysis = configSetHelper.analyzeField(configSet, fieldName, textValue);
var analysis = configSetHelper.analyzeField(configSet, fieldName, textValue);
rsp.getValues().addAll(Map.of(idField, docId, fieldName, textValue, "analysis", analysis));
}
}
Expand Down
150 changes: 38 additions & 112 deletions solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.solr.handler.designer;

import static org.apache.solr.common.params.CommonParams.VERSION_FIELD;
import static org.apache.solr.common.util.Utils.fromJSONString;
import static org.apache.solr.common.util.Utils.toJavabin;
import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
import static org.apache.solr.handler.designer.SchemaDesignerAPI.getConfigSetZkPath;
Expand All @@ -31,8 +30,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
Expand All @@ -57,33 +54,30 @@
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.file.PathUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.util.EntityUtils;
import org.apache.lucene.util.IOSupplier;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.cloud.ZkConfigSetService;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
Expand Down Expand Up @@ -124,46 +118,22 @@ class SchemaDesignerConfigSetHelper implements SchemaDesignerConstants {
}

@SuppressWarnings("unchecked")
Map<String, Object> analyzeField(String configSet, String fieldName, String fieldText)
NamedList<Object> analyzeField(String configSet, String fieldName, String fieldText)
iamsanjay marked this conversation as resolved.
Show resolved Hide resolved
throws IOException {
final String mutableId = getMutableId(configSet);
final URI uri;
var solrParams = new ModifiableSolrParams();
solrParams.add("analysis.showmatch", "true");
solrParams.add("analysis.fieldname", fieldName);
solrParams.add("analysis.fieldvalue", "POST");
var request = new GenericSolrRequest(SolrRequest.METHOD.POST, "/analysis/field", solrParams);
request.withContent(fieldText.getBytes(StandardCharsets.UTF_8), "text/plain");
request.setRequiresCollection(true);
try {
uri =
collectionApiEndpoint(mutableId, "analysis", "field")
.setParameter(CommonParams.WT, CommonParams.JSON)
.setParameter("analysis.showmatch", "true")
.setParameter("analysis.fieldname", fieldName)
.setParameter("analysis.fieldvalue", "POST")
.build();
} catch (URISyntaxException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
var resp = request.process(cloudClient(), mutableId).getResponse();
return (NamedList<Object>) resp.get("analysis");
} catch (SolrServerException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}

Map<String, Object> analysis = Collections.emptyMap();
HttpPost httpPost = new HttpPost(uri);
httpPost.setHeader("Content-Type", "text/plain");
httpPost.setEntity(new ByteArrayEntity(fieldText.getBytes(StandardCharsets.UTF_8)));
try {
HttpResponse resp = ((CloudLegacySolrClient) cloudClient()).getHttpClient().execute(httpPost);
int statusCode = resp.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new SolrException(
SolrException.ErrorCode.getErrorCode(statusCode),
EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
}

Map<String, Object> response =
(Map<String, Object>)
fromJSONString(EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
if (response != null) {
analysis = (Map<String, Object>) response.get("analysis");
}
} finally {
httpPost.releaseConnection();
}

return analysis;
}

List<String> listCollectionsForConfig(String configSet) {
Expand Down Expand Up @@ -511,40 +481,22 @@ void deleteStoredSampleDocs(String configSet) {

@SuppressWarnings("unchecked")
List<SolrInputDocument> getStoredSampleDocs(final String configSet) throws IOException {
List<SolrInputDocument> docs = null;

final URI uri;
try {
uri =
collectionApiEndpoint(BLOB_STORE_ID, "blob", configSet + "_sample")
.setParameter(CommonParams.WT, "filestream")
.build();
} catch (URISyntaxException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}

HttpGet httpGet = new HttpGet(uri);
var request = new GenericSolrRequest(SolrRequest.METHOD.GET, "/blob/" + configSet + "_sample");
request.setRequiresCollection(true);
request.setResponseParser(new InputStreamResponseParser("filestream"));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I played with this method a bit, hoping to remove InputStream low level stuff but there seems to be something special about "filestream" so I left this aspect as is. Besides, the response handling is pretty concise nonetheless.

InputStream inputStream = null;
try {
HttpResponse entity =
((CloudLegacySolrClient) cloudClient()).getHttpClient().execute(httpGet);
int statusCode = entity.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
byte[] bytes = readAllBytes(() -> entity.getEntity().getContent());
if (bytes.length > 0) {
docs = (List<SolrInputDocument>) Utils.fromJavabin(bytes);
}
} else if (statusCode != HttpStatus.SC_NOT_FOUND) {
byte[] bytes = readAllBytes(() -> entity.getEntity().getContent());
throw new IOException(
"Failed to lookup stored docs for "
+ configSet
+ " due to: "
+ new String(bytes, StandardCharsets.UTF_8));
} // else not found is ok
var resp = request.process(cloudClient(), BLOB_STORE_ID).getResponse();
inputStream = (InputStream) resp.get("stream");
var bytes = inputStream.readAllBytes();
if (bytes.length > 0) {
return (List<SolrInputDocument>) Utils.fromJavabin(bytes);
} else return Collections.emptyList();
} catch (SolrServerException e) {
throw new IOException("Failed to lookup stored docs for " + configSet + " due to: " + e);
} finally {
httpGet.releaseConnection();
IOUtils.closeQuietly(inputStream);
}
return docs != null ? docs : Collections.emptyList();
}

void storeSampleDocs(final String configSet, List<SolrInputDocument> docs) throws IOException {
Expand All @@ -561,26 +513,13 @@ static byte[] readAllBytes(IOSupplier<InputStream> hasStream) throws IOException

protected void postDataToBlobStore(CloudSolrClient cloudClient, String blobName, byte[] bytes)
throws IOException {
final URI uri;
try {
uri = collectionApiEndpoint(BLOB_STORE_ID, "blob", blobName).build();
} catch (URISyntaxException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}

HttpPost httpPost = new HttpPost(uri);
var request = new GenericSolrRequest(SolrRequest.METHOD.POST, "/blob/" + blobName);
request.withContent(bytes, BinaryResponseParser.BINARY_CONTENT_TYPE);
request.setRequiresCollection(true);
try {
httpPost.setHeader("Content-Type", "application/octet-stream");
httpPost.setEntity(new ByteArrayEntity(bytes));
HttpResponse resp = ((CloudLegacySolrClient) cloudClient).getHttpClient().execute(httpPost);
int statusCode = resp.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new SolrException(
SolrException.ErrorCode.getErrorCode(statusCode),
EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
}
} finally {
httpPost.releaseConnection();
request.process(cloudClient, BLOB_STORE_ID);
} catch (SolrServerException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
}

Expand All @@ -607,19 +546,6 @@ private String getBaseUrl(final String collection) {
return baseUrl;
}

private URIBuilder collectionApiEndpoint(
final String collection, final String... morePathSegments) throws URISyntaxException {
URI baseUrl = new URI(getBaseUrl(collection));
// build up a list of path segments including any path in the base URL, collection, and
// additional segments provided by caller
List<String> path = new ArrayList<>(URLEncodedUtils.parsePathSegments(baseUrl.getPath()));
path.add(collection);
if (morePathSegments != null && morePathSegments.length > 0) {
path.addAll(Arrays.asList(morePathSegments));
}
return new URIBuilder(baseUrl).setPathSegments(path);
}

protected String getManagedSchemaZkPath(final String configSet) {
return getConfigSetZkPath(configSet, DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME);
}
Expand Down
11 changes: 4 additions & 7 deletions solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
Expand Down Expand Up @@ -280,15 +281,11 @@ public void testAnalyzeField() throws Exception {
helper.addSchemaObject(configSet, Collections.singletonMap("add-field", addField));
assertEquals("title", addedFieldName);

Map<String, Object> analysis =
NamedList<Object> analysis =
helper.analyzeField(configSet, "title", "The Pillars of the Earth");

Map<String, Object> title =
(Map<String, Object>) ((Map<String, Object>) analysis.get("field_names")).get("title");
assertNotNull(title);
List<Object> index = (List<Object>) title.get("index");
assertNotNull(index);
assertFalse(index.isEmpty());
var indexNl = (NamedList<Object>) analysis.findRecursive("field_names", "title", "index");
iamsanjay marked this conversation as resolved.
Show resolved Hide resolved
assertTrue(indexNl.size() > 0);
}

@Test
Expand Down
Loading