Skip to content

Commit

Permalink
Merge branch 'main' into SOLR-17195-min-prefix-soft-limit
Browse files Browse the repository at this point in the history
  • Loading branch information
gerlowskija committed Jul 9, 2024
2 parents 85d1546 + 9a86b21 commit 36b88f1
Show file tree
Hide file tree
Showing 61 changed files with 4,326 additions and 175 deletions.
4 changes: 4 additions & 0 deletions solr/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ New Features

* SOLR-17335: New "vectorSimilarity" QParser for matching documents mased on a minimum vector similarity threshold. (hossman)

* SOLR-10654: Introduce output of Prometheus metrics directly from Solr. (Matthew Biscocho via David Smiley)

* SOLR-17195: Configsets now include a `minPrefixQueryTermLength` setting, which instructs Solr to reject prefix queries whose prefixes are "too short". This can
be used as one line of defense against "runaway wildcard queries" consuming too many resources. The setting is disabled ('-1') in the default configset but can be
overridden with a property ('solr.query.minPrefixLength'). Users may also override their collection-wide setting for individual queries by providing a
Expand Down Expand Up @@ -188,6 +190,8 @@ Bug Fixes

* SOLR-17255: Fix bugs in SolrParams.toLocalParamsString() (hossman)

* SOLR-17333: Rate-limiting feature: fix live-update of config (Michael Gibney)

Dependency Upgrades
---------------------
(No changes)
Expand Down
7 changes: 7 additions & 0 deletions solr/core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,13 @@ dependencies {
implementation 'org.apache.logging.log4j:log4j-core'
runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl'

// For the PrometheusResponseWriter
implementation 'io.prometheus:prometheus-metrics-model:1.1.0'
implementation('io.prometheus:prometheus-metrics-exposition-formats:1.1.0', {
exclude group: "io.prometheus", module: "prometheus-metrics-shaded-protobuf"
exclude group: "io.prometheus", module: "prometheus-metrics-config"
})

// For faster XML processing than the JDK
implementation 'org.codehaus.woodstox:stax2-api'
implementation 'com.fasterxml.woodstox:woodstox-core'
Expand Down
93 changes: 76 additions & 17 deletions solr/core/src/java/org/apache/solr/core/RateLimiterConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,26 @@
import static org.apache.solr.servlet.RateLimitManager.DEFAULT_SLOT_ACQUISITION_TIMEOUT_MS;

import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.beans.RateLimiterPayload;

public class RateLimiterConfig {
public static final String RL_CONFIG_KEY = "rate-limiters";

public SolrRequest.SolrRequestType requestType;
public boolean isEnabled;
public long waitForSlotAcquisition;
public int allowedRequests;
public boolean isSlotBorrowingEnabled;
public int guaranteedSlotsThreshold;
public final SolrRequest.SolrRequestType requestType;
public final boolean isEnabled;
public final long waitForSlotAcquisition;
public final int allowedRequests;
public final boolean isSlotBorrowingEnabled;
public final int guaranteedSlotsThreshold;

/**
* We store the config definition in order to determine whether anything has changed that would
* call for re-initialization.
*/
public final RateLimiterPayload definition;

public RateLimiterConfig(SolrRequest.SolrRequestType requestType) {
this.requestType = requestType;
this.isEnabled = false;
this.allowedRequests = DEFAULT_CONCURRENT_REQUESTS;
this.isSlotBorrowingEnabled = false;
this.guaranteedSlotsThreshold = this.allowedRequests / 2;
this.waitForSlotAcquisition = DEFAULT_SLOT_ACQUISITION_TIMEOUT_MS;
this(requestType, EMPTY);
}

public RateLimiterConfig(
Expand All @@ -48,11 +50,68 @@ public RateLimiterConfig(
long waitForSlotAcquisition,
int allowedRequests,
boolean isSlotBorrowingEnabled) {
this(
requestType,
makePayload(
isEnabled,
guaranteedSlotsThreshold,
waitForSlotAcquisition,
allowedRequests,
isSlotBorrowingEnabled));
}

private static RateLimiterPayload makePayload(
boolean isEnabled,
int guaranteedSlotsThreshold,
long waitForSlotAcquisition,
int allowedRequests,
boolean isSlotBorrowingEnabled) {
RateLimiterPayload ret = new RateLimiterPayload();
ret.enabled = isEnabled;
ret.allowedRequests = allowedRequests;
ret.guaranteedSlots = guaranteedSlotsThreshold;
ret.slotBorrowingEnabled = isSlotBorrowingEnabled;
ret.slotAcquisitionTimeoutInMS = Math.toIntExact(waitForSlotAcquisition);
return ret;
}

public RateLimiterConfig(SolrRequest.SolrRequestType requestType, RateLimiterPayload definition) {
this.requestType = requestType;
this.isEnabled = isEnabled;
this.guaranteedSlotsThreshold = guaranteedSlotsThreshold;
this.waitForSlotAcquisition = waitForSlotAcquisition;
this.allowedRequests = allowedRequests;
this.isSlotBorrowingEnabled = isSlotBorrowingEnabled;
if (definition == null) {
definition = EMPTY;
}
allowedRequests =
definition.allowedRequests == null
? DEFAULT_CONCURRENT_REQUESTS
: definition.allowedRequests;

isEnabled = definition.enabled == null ? false : definition.enabled; // disabled by default

guaranteedSlotsThreshold =
definition.guaranteedSlots == null ? this.allowedRequests / 2 : definition.guaranteedSlots;

isSlotBorrowingEnabled =
definition.slotBorrowingEnabled == null ? false : definition.slotBorrowingEnabled;

waitForSlotAcquisition =
definition.slotAcquisitionTimeoutInMS == null
? DEFAULT_SLOT_ACQUISITION_TIMEOUT_MS
: definition.slotAcquisitionTimeoutInMS.longValue();

this.definition = definition;
}

private static final RateLimiterPayload EMPTY = new RateLimiterPayload(); // use defaults;

public boolean shouldUpdate(RateLimiterPayload definition) {
if (definition == null) {
definition = EMPTY; // use defaults
}

if (definition.equals(this.definition)) {
return false;
}

return true;
}
}
3 changes: 3 additions & 0 deletions solr/core/src/java/org/apache/solr/core/SolrCore.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.apache.solr.core;

import static org.apache.solr.common.params.CommonParams.PATH;
import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT;

import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
Expand Down Expand Up @@ -129,6 +130,7 @@
import org.apache.solr.response.JacksonJsonWriter;
import org.apache.solr.response.PHPResponseWriter;
import org.apache.solr.response.PHPSerializedResponseWriter;
import org.apache.solr.response.PrometheusResponseWriter;
import org.apache.solr.response.PythonResponseWriter;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.RawResponseWriter;
Expand Down Expand Up @@ -3018,6 +3020,7 @@ public PluginBag<QueryResponseWriter> getResponseWriters() {
m.put("csv", new CSVResponseWriter());
m.put("schema.xml", new SchemaXmlResponseWriter());
m.put("smile", new SmileResponseWriter());
m.put(PROMETHEUS_METRICS_WT, new PrometheusResponseWriter());
m.put(ReplicationHandler.FILE_STREAM, getFileStreamWriter());
DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.util.stream.Collectors;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.CommonTestInjection;
import org.apache.solr.common.util.NamedList;
Expand All @@ -50,6 +51,7 @@
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.PrometheusResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
Expand All @@ -68,6 +70,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
public static final String KEY_PARAM = "key";
public static final String EXPR_PARAM = "expr";
public static final String TYPE_PARAM = "type";
public static final String PROMETHEUS_METRICS_WT = "prometheus";

public static final String ALL = "all";

Expand Down Expand Up @@ -116,13 +119,19 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw
}
}

private void handleRequest(SolrParams params, BiConsumer<String, Object> consumer)
throws Exception {
private void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) {
NamedList<Object> response;

if (!enabled) {
consumer.accept("error", "metrics collection is disabled");
return;
}
boolean compact = params.getBool(COMPACT_PARAM, true);

if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))) {
response = handlePrometheusExport(params);
consumer.accept("metrics", response);
return;
}
String[] keys = params.getParams(KEY_PARAM);
if (keys != null && keys.length > 0) {
handleKeyRequest(keys, consumer);
Expand All @@ -133,6 +142,14 @@ private void handleRequest(SolrParams params, BiConsumer<String, Object> consume
handleExprRequest(exprs, consumer);
return;
}

response = handleDropwizardRegistry(params);

consumer.accept("metrics", response);
}

private NamedList<Object> handleDropwizardRegistry(SolrParams params) {
boolean compact = params.getBool(COMPACT_PARAM, true);
MetricFilter mustMatchFilter = parseMustMatchFilter(params);
Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);
List<MetricType> metricTypes = parseMetricTypes(params);
Expand All @@ -144,6 +161,7 @@ private void handleRequest(SolrParams params, BiConsumer<String, Object> consume
for (String registryName : requestedRegistries) {
MetricRegistry registry = metricManager.registry(registryName);
SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();

MetricUtils.toMaps(
registry,
metricFilters,
Expand All @@ -158,7 +176,34 @@ private void handleRequest(SolrParams params, BiConsumer<String, Object> consume
response.add(registryName, result);
}
}
consumer.accept("metrics", response);
return response;
}

private NamedList<Object> handlePrometheusExport(SolrParams params) {
NamedList<Object> response = new SimpleOrderedMap<>();
MetricFilter mustMatchFilter = parseMustMatchFilter(params);
Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);
List<MetricType> metricTypes = parseMetricTypes(params);
List<MetricFilter> metricFilters =
metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());
Set<String> requestedRegistries = parseRegistries(params);

for (String registryName : requestedRegistries) {
MetricRegistry dropwizardRegistry = metricManager.registry(registryName);
PrometheusResponseWriter.toPrometheus(
dropwizardRegistry,
registryName,
metricFilters,
mustMatchFilter,
propertyFilter,
false,
false,
true,
(registry) -> {
response.add(registryName, registry);
});
}
return response;
}

private static class MetricsExpr {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.metrics.prometheus;

import com.codahale.metrics.Metric;
import io.prometheus.metrics.model.snapshots.Labels;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

/**
* Base class is a wrapper to categorize and export {@link com.codahale.metrics.Metric} to {@link
* io.prometheus.metrics.model.snapshots.DataPointSnapshot} and register to a {@link
* SolrPrometheusExporter}. {@link com.codahale.metrics.MetricRegistry} does not support tags unlike
* prometheus. Metrics registered to the registry need to be parsed out from the metric name to be
* exported to {@link io.prometheus.metrics.model.snapshots.DataPointSnapshot}
*/
public abstract class SolrMetric {
public Metric dropwizardMetric;
public String metricName;
public Map<String, String> labels = new HashMap<>();

public SolrMetric() {}

public SolrMetric(Metric dropwizardMetric, String metricName) {
this.dropwizardMetric = dropwizardMetric;
this.metricName = metricName;
}

/*
* Parse labels from the Dropwizard Metric name to be exported
*/
public abstract SolrMetric parseLabels();

/*
* Export metric to Prometheus with labels
*/
public abstract void toPrometheus(SolrPrometheusExporter exporter);

public Labels getLabels() {
return Labels.of(new ArrayList<>(labels.keySet()), new ArrayList<>(labels.values()));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.metrics.prometheus;

public class SolrNoOpMetric extends SolrMetric {
public SolrNoOpMetric() {}

@Override
public SolrMetric parseLabels() {
return this;
}

@Override
public void toPrometheus(SolrPrometheusExporter exporter) {}
}
Loading

0 comments on commit 36b88f1

Please sign in to comment.