SOLR-10654: Introduce output of Prometheus metrics directly from Solr

solr/core/build.gradle

@@ -153,6 +153,11 @@ dependencies {
   implementation 'org.apache.logging.log4j:log4j-core'
   runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl'
 
+  // Prometheus client
+  implementation 'io.prometheus:prometheus-metrics-core:1.1.0'
+  implementation 'io.prometheus:prometheus-metrics-model:1.1.0'
+  implementation 'io.prometheus:prometheus-metrics-exposition-formats:1.1.0'
+
   // For faster XML processing than the JDK
   implementation 'org.codehaus.woodstox:stax2-api'
   implementation 'com.fasterxml.woodstox:woodstox-core'

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -17,6 +17,7 @@
 package org.apache.solr.core;
 
 import static org.apache.solr.common.params.CommonParams.PATH;
+import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT;
 
 import com.codahale.metrics.Counter;
 import com.codahale.metrics.Timer;
@@ -130,6 +131,7 @@
 import org.apache.solr.response.JacksonJsonWriter;
 import org.apache.solr.response.PHPResponseWriter;
 import org.apache.solr.response.PHPSerializedResponseWriter;
+import org.apache.solr.response.PrometheusResponseWriter;
 import org.apache.solr.response.PythonResponseWriter;
 import org.apache.solr.response.QueryResponseWriter;
 import org.apache.solr.response.RawResponseWriter;
@@ -3032,6 +3034,7 @@ public PluginBag<QueryResponseWriter> getResponseWriters() {
     m.put("csv", new CSVResponseWriter());
     m.put("schema.xml", new SchemaXmlResponseWriter());
     m.put("smile", new SmileResponseWriter());
+    m.put(PROMETHEUS_METRICS_WT, new PrometheusResponseWriter());
     m.put(ReplicationHandler.FILE_STREAM, getFileStreamWriter());
     DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);
     try {

solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java

@@ -40,6 +40,7 @@
 import java.util.stream.Collectors;
 import org.apache.solr.common.MapWriter;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.CommonTestInjection;
 import org.apache.solr.common.util.NamedList;
@@ -68,6 +69,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
   public static final String KEY_PARAM = "key";
   public static final String EXPR_PARAM = "expr";
   public static final String TYPE_PARAM = "type";
+  public static final String PROMETHEUS_METRICS_WT = "prometheus";
 
   public static final String ALL = "all";
 
@@ -118,11 +120,18 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw
 
   private void handleRequest(SolrParams params, BiConsumer<String, Object> consumer)
       throws Exception {
+    NamedList<Object> response;
+
     if (!enabled) {
       consumer.accept("error", "metrics collection is disabled");
       return;
     }
-    boolean compact = params.getBool(COMPACT_PARAM, true);
+
+    if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))) {
+      response = handlePrometheusRegistry(params);
+      consumer.accept("metrics", response);
+      return;
+    }
     String[] keys = params.getParams(KEY_PARAM);
     if (keys != null && keys.length > 0) {
       handleKeyRequest(keys, consumer);
@@ -133,6 +142,14 @@ private void handleRequest(SolrParams params, BiConsumer<String, Object> consume
       handleExprRequest(exprs, consumer);
       return;
     }
+
+    response = handleDropwizardRegistry(params);
+
+    consumer.accept("metrics", response);
+  }
+
+  private NamedList<Object> handleDropwizardRegistry(SolrParams params) {
+    boolean compact = params.getBool(COMPACT_PARAM, true);
     MetricFilter mustMatchFilter = parseMustMatchFilter(params);
     Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);
     List<MetricType> metricTypes = parseMetricTypes(params);
@@ -144,6 +161,7 @@ private void handleRequest(SolrParams params, BiConsumer<String, Object> consume
     for (String registryName : requestedRegistries) {
       MetricRegistry registry = metricManager.registry(registryName);
       SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
+
       MetricUtils.toMaps(
           registry,
           metricFilters,
@@ -158,7 +176,38 @@ private void handleRequest(SolrParams params, BiConsumer<String, Object> consume
         response.add(registryName, result);
       }
     }
-    consumer.accept("metrics", response);
+    return response;
+  }
+
+  private NamedList<Object> handlePrometheusRegistry(SolrParams params) {
+    NamedList<Object> response = new SimpleOrderedMap<>();
+    boolean compact = params.getBool(COMPACT_PARAM, true);
+    MetricFilter mustMatchFilter = parseMustMatchFilter(params);
+    Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);
+    List<MetricType> metricTypes = parseMetricTypes(params);
+    List<MetricFilter> metricFilters =
+        metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());
+    Set<String> requestedRegistries = parseRegistries(params);
+
+    for (String registryName : requestedRegistries) {
+      MetricRegistry dropwizardRegistry = metricManager.registry(registryName);
+      // Currently only export Solr Core registries
+      if (registryName.startsWith("solr.core")) {
+        MetricUtils.toPrometheusRegistry(
+            dropwizardRegistry,
+            registryName,
+            metricFilters,
+            mustMatchFilter,
+            propertyFilter,
+            false,
+            false,
+            compact,
+            (registry) -> {
+              response.add(registryName, registry);
+            });
+      }
+    }
+    return response;
   }
 
   private static class MetricsExpr {

solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusCoreRegistry.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.prometheus;
+
+import com.codahale.metrics.Metric;
+import io.prometheus.metrics.model.registry.PrometheusRegistry;
+import org.apache.solr.metrics.prometheus.core.SolrCoreCacheMetric;
+import org.apache.solr.metrics.prometheus.core.SolrCoreHandlerMetric;
+import org.apache.solr.metrics.prometheus.core.SolrCoreHighlighterMetric;
+import org.apache.solr.metrics.prometheus.core.SolrCoreIndexMetric;
+import org.apache.solr.metrics.prometheus.core.SolrCoreMetric;
+import org.apache.solr.metrics.prometheus.core.SolrCoreNoOpMetric;
+import org.apache.solr.metrics.prometheus.core.SolrCoreSearcherMetric;
+import org.apache.solr.metrics.prometheus.core.SolrCoreTlogMetric;
+
+public class SolrPrometheusCoreRegistry extends SolrPrometheusRegistry {
+  public final String coreName;
+  public final boolean cloudMode;
+  public static final String ADMIN = "ADMIN";
+  public static final String QUERY = "QUERY";
+  public static final String UPDATE = "UPDATE";
+  public static final String REPLICATION = "REPLICATION";
+  public static final String TLOG = "TLOG";
+  public static final String CACHE = "CACHE";
+  public static final String SEARCHER = "SEARCHER";
+  public static final String HIGHLIGHTER = "HIGHLIGHTER";
+  public static final String INDEX = "INDEX";
+  public static final String CORE = "CORE";
+
+  public SolrPrometheusCoreRegistry(
+      PrometheusRegistry prometheusRegistry, String coreName, boolean cloudMode) {
+    super(prometheusRegistry);
+    this.coreName = coreName;
+    this.cloudMode = cloudMode;
+  }
+
+  public void exportDropwizardMetric( Metric dropwizardMetric, String metricName) {
+    SolrCoreMetric solrCoreMetric = categorizeCoreMetric(dropwizardMetric, metricName);
+    solrCoreMetric.parseLabels().toPrometheus(this);
+  }
+
+  private SolrCoreMetric categorizeCoreMetric(Metric dropwizardMetric, String metricName) {
+    String metricCategory = metricName.split("\\.")[0];
+    switch (metricCategory) {
+      case ADMIN:
+      case QUERY:
+      case UPDATE:
+      case REPLICATION:
+        {
+          return new SolrCoreHandlerMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        }
+      case TLOG:
+        {
+          return new SolrCoreTlogMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        }
+      case CACHE:
+        {
+          return new SolrCoreCacheMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        }
+      case SEARCHER:
+        {
+          return new SolrCoreSearcherMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        }
+      case HIGHLIGHTER:
+        {
+          return new SolrCoreHighlighterMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        }
+      case INDEX:
+        {
+          return new SolrCoreIndexMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        }
+      case CORE:
+      default:
+        {
+          return new SolrCoreNoOpMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        }
+    }
+  }
+}

solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusRegistry.java

@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.prometheus;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import io.prometheus.metrics.core.metrics.Counter;
+import io.prometheus.metrics.core.metrics.Gauge;
+import io.prometheus.metrics.model.registry.PrometheusRegistry;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class SolrPrometheusRegistry {
+  PrometheusRegistry prometheusRegistry;
+  String registryName;
+  private final Map<String, Counter> metricCounters;
+  private final Map<String, Gauge> metricGauges;
+
+  public SolrPrometheusRegistry(PrometheusRegistry prometheusRegistry) {
+    this.prometheusRegistry = prometheusRegistry;
+    this.metricCounters = new HashMap<>();
+    this.metricGauges = new HashMap<>();
+  }
+
+  public PrometheusRegistry getPrometheusRegistry() {
+    return prometheusRegistry;
+  }
+
+  public String getRegistryName() {
+    return registryName;
+  }
+
+  public void exportMeter(
+      Meter dropwizardMetric, String prometheusMetricName, Map<String, String> labelsMap) {
+    if (!metricCounters.containsKey(prometheusMetricName)) {
+      ArrayList<String> labels = new ArrayList<>(labelsMap.keySet());
+      registerCounter(prometheusMetricName, labels.toArray(String[]::new));
+    }
+    ArrayList<String> labelValues = new ArrayList<>(labelsMap.values());
+    getMetricCounter(prometheusMetricName)
+        .labelValues(labelValues.toArray(String[]::new))
+        .inc(dropwizardMetric.getCount());
+  }
+
+  public void exportCounter(
+      com.codahale.metrics.Counter dropwizardMetric,
+      String prometheusMetricName,
+      Map<String, String> labelsMap) {
+    if (!metricCounters.containsKey(prometheusMetricName)) {
+      ArrayList<String> labels = new ArrayList<>(labelsMap.keySet());
+      registerCounter(prometheusMetricName, labels.toArray(String[]::new));
+    }
+    ArrayList<String> labelValues = new ArrayList<>(labelsMap.values());
+    getMetricCounter(prometheusMetricName)
+        .labelValues(labelValues.toArray(String[]::new))
+        .inc(dropwizardMetric.getCount());
+  }
+
+  public void exportTimer(
+      Timer dropwizardMetric, String prometheusMetricName, Map<String, String> labelsMap) {
+    if (!metricGauges.containsKey(prometheusMetricName)) {
+      ArrayList<String> labels = new ArrayList<>(labelsMap.keySet());
+      registerGauge(prometheusMetricName, labels.toArray(String[]::new));
+    }
+    ArrayList<String> labelValues = new ArrayList<>(labelsMap.values());
+    getMetricGauge(prometheusMetricName)
+        .labelValues(labelValues.toArray(String[]::new))
+        .set(dropwizardMetric.getMeanRate());
+  }
+
+  public void exportGauge(
+      com.codahale.metrics.Gauge<?> dropwizardMetricRaw,
+      String prometheusMetricName,
+      Map<String, String> labelsMap) {
+    Object dropwizardMetric = (dropwizardMetricRaw).getValue();
+    if (!metricGauges.containsKey(prometheusMetricName)) {
+      ArrayList<String> labels = new ArrayList<>(labelsMap.keySet());
+      if (dropwizardMetric instanceof HashMap) {
+        labels.add("item");
+      }
+      registerGauge(prometheusMetricName, labels.toArray(String[]::new));
+    }
+    ArrayList<String> labelValues = new ArrayList<>(labelsMap.values());
+    String[] labels = labelValues.toArray(String[]::new);
+    if (dropwizardMetric instanceof Number) {
+      getMetricGauge(prometheusMetricName)
+          .labelValues(labels)
+          .set(((Number) dropwizardMetric).doubleValue());
+    } else if (dropwizardMetric instanceof HashMap) {
+      HashMap<?, ?> itemsMap = (HashMap<?, ?>) dropwizardMetric;
+      for (Object item : itemsMap.keySet()) {
+        if (itemsMap.get(item) instanceof Number) {
+          String[] newLabels = new String[labels.length + 1];
+          System.arraycopy(labels, 0, newLabels, 0, labels.length);
+          newLabels[labels.length] = (String) item;
+          getMetricGauge(prometheusMetricName)
+              .labelValues(newLabels)
+              .set(((Number) itemsMap.get(item)).doubleValue());
+        }
+      }
+    }
+  }
+
+  private Counter getMetricCounter(String metricName) {
+    return metricCounters.get(metricName);
+  }
+
+  private Gauge getMetricGauge(String metricName) {
+    return metricGauges.get(metricName);
+  }
+
+  private void registerCounter(String metricName, String... labelNames) {
+    Counter counter =
+        io.prometheus.metrics.core.metrics.Counter.builder()
+            .name(metricName)
+            .labelNames(labelNames)
+            .register(prometheusRegistry);
+    metricCounters.put(metricName, counter);
+  }
+
+  private void registerGauge(String metricName, String... labelNames) {
+    Gauge gauge =
+        io.prometheus.metrics.core.metrics.Gauge.builder()
+            .name(metricName)
+            .labelNames(labelNames)
+            .register(prometheusRegistry);
+    metricGauges.put(metricName, gauge);
+  }
+}