Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add druid_service and host_name labels to Prometheus exporter #12769

Merged
merged 5 commits into from
Jul 15, 2022
Merged

Add druid_service and host_name labels to Prometheus exporter #12769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7d7f141
Add druid_service and host_name labels to Prometheus exporter
liujianhuanzz Jul 11, 2022
99dd1e3
fix checkstyle
liujianhuanzz Jul 11, 2022
d1bebd8
modify code by review
liujianhuanzz Jul 13, 2022
5ea407a
checkstyle
liujianhuanzz Jul 13, 2022
aaf2d1c
modify docs
liujianhuanzz Jul 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/development/extensions-contrib/prometheus.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ All the configuration parameters for the Prometheus emitter are under `druid.emi
|`druid.emitter.prometheus.port`|The port on which to expose the prometheus HTTPServer. Required if using exporter strategy.|no|none|
|`druid.emitter.prometheus.namespace`|Optional metric namespace. Must match the regex `[a-zA-Z_:][a-zA-Z0-9_:]*`|no|"druid"|
|`druid.emitter.prometheus.dimensionMapPath`|JSON file defining the Prometheus metric type, desired dimensions, help text, and conversionFactor for every Druid metric.|no|Default mapping provided. See below.|
|`druid.emitter.prometheus.addHostAsLabel`|Flag to include the hostname as a prometheus label.|no|false|
|`druid.emitter.prometheus.addServiceAsLabel`|Flag to include the druid service name (e.g. `druid/broker`, `druid/coordinator`, etc.) as a prometheus label.|no|false|
|`druid.emitter.prometheus.pushGatewayAddress`|Pushgateway address. Required if using Pushgateway strategy|no|none|


Expand Down
6 changes: 3 additions & 3 deletions extensions-contrib/prometheus-emitter/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,17 @@
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
<version>0.7.0</version>
<version>0.16.0</version>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks!

I assume this version is fully backward compatible and doesn't cause any upgrade issues.

</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
<version>0.7.0</version>
<version>0.16.0</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_pushgateway</artifactId>
<version>0.7.0</version>
<version>0.16.0</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
Expand Down
14 changes: 13 additions & 1 deletion ...contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/Metrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ public class Metrics
private final ObjectMapper mapper = new ObjectMapper();
public static final Pattern PATTERN = Pattern.compile("[^a-zA-Z_:][^a-zA-Z0-9_:]*");

private static final String TAG_HOSTNAME = "host_name";
private static final String TAG_SERVICE = "druid_service";

public DimensionsAndCollector getByName(String name, String service)
{
if (registeredMetrics.containsKey(name)) {
Expand All @@ -58,13 +61,22 @@ public DimensionsAndCollector getByName(String name, String service)
}
}

public Metrics(String namespace, String path)
public Metrics(String namespace, String path, boolean isAddHostAsLabel, boolean isAddServiceAsLabel)
{
Map<String, Metric> metrics = readConfig(path);
for (Map.Entry<String, Metric> entry : metrics.entrySet()) {
String name = entry.getKey();
Metric metric = entry.getValue();
Metric.Type type = metric.type;

if (isAddHostAsLabel) {
metric.dimensions.add(TAG_HOSTNAME);
}

if (isAddServiceAsLabel) {
metric.dimensions.add(TAG_SERVICE);
}

String[] dimensions = metric.dimensions.toArray(new String[0]);
String formattedName = PATTERN.matcher(StringUtils.toLowerCase(name)).replaceAll("_");
SimpleCollector collector = null;
Expand Down
19 changes: 17 additions & 2 deletions ...ometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ public class PrometheusEmitter implements Emitter
private final PrometheusEmitterConfig.Strategy strategy;
private static final Pattern PATTERN = Pattern.compile("[^a-zA-Z0-9_][^a-zA-Z0-9_]*");

private static final String TAG_HOSTNAME = "host_name";
private static final String TAG_SERVICE = "druid_service";

private HTTPServer server;
private PushGateway pushGateway;
private String identifier;
Expand All @@ -62,7 +65,7 @@ public PrometheusEmitter(PrometheusEmitterConfig config)
{
this.config = config;
this.strategy = config.getStrategy();
metrics = new Metrics(config.getNamespace(), config.getDimensionMapPath());
metrics = new Metrics(config.getNamespace(), config.getDimensionMapPath(), config.isAddHostAsLabel(), config.isAddServiceAsLabel());
}


Expand Down Expand Up @@ -113,6 +116,7 @@ private void emitMetric(ServiceMetricEvent metricEvent)
{
String name = metricEvent.getMetric();
String service = metricEvent.getService();
String host = metricEvent.getHost();
Map<String, Object> userDims = metricEvent.getUserDims();
identifier = (userDims.get("task") == null ? metricEvent.getHost() : (String) userDims.get("task"));
Number value = metricEvent.getValue();
Expand All @@ -125,7 +129,18 @@ private void emitMetric(ServiceMetricEvent metricEvent)
String labelName = labelNames[i];
//labelName is controlled by the user. Instead of potential NPE on invalid labelName we use "unknown" as the dimension value
Object userDim = userDims.get(labelName);
labelValues[i] = userDim != null ? PATTERN.matcher(userDim.toString()).replaceAll("_") : "unknown";

if (userDim != null) {
labelValues[i] = PATTERN.matcher(userDim.toString()).replaceAll("_");
} else {
if (config.isAddHostAsLabel() && TAG_HOSTNAME.equals(labelName)) {
labelValues[i] = host;
} else if (config.isAddServiceAsLabel() && TAG_SERVICE.equals(labelName)) {
labelValues[i] = service;
} else {
labelValues[i] = "unknown";
}
}
}

if (metric.getCollector() instanceof Counter) {
Expand Down
22 changes: 21 additions & 1 deletion ...us-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitterConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,21 @@ public class PrometheusEmitterConfig
@Nullable
private final String pushGatewayAddress;

@JsonProperty
private final boolean addHostAsLabel;

@JsonProperty
private final boolean addServiceAsLabel;

@JsonCreator
public PrometheusEmitterConfig(
@JsonProperty("strategy") @Nullable Strategy strategy,
@JsonProperty("namespace") @Nullable String namespace,
@JsonProperty("dimensionMapPath") @Nullable String dimensionMapPath,
@JsonProperty("port") @Nullable Integer port,
@JsonProperty("pushGatewayAddress") @Nullable String pushGatewayAddress
@JsonProperty("pushGatewayAddress") @Nullable String pushGatewayAddress,
@JsonProperty("addHostAsLabel") boolean addHostAsLabel,
@JsonProperty("addServiceAsLabel") boolean addServiceAsLabel
)
{

Expand All @@ -72,6 +80,8 @@ public PrometheusEmitterConfig(
Preconditions.checkNotNull(pushGatewayAddress, "Invalid pushGateway address");
}
this.pushGatewayAddress = pushGatewayAddress;
this.addHostAsLabel = addHostAsLabel;
this.addServiceAsLabel = addServiceAsLabel;
}

public String getNamespace()
Expand Down Expand Up @@ -99,6 +109,16 @@ public Strategy getStrategy()
return strategy;
}

public boolean isAddHostAsLabel()
{
return addHostAsLabel;
}

public boolean isAddServiceAsLabel()
{
return addServiceAsLabel;
}

public enum Strategy
{
exporter, pushgateway
Expand Down
10 changes: 7 additions & 3 deletions ...rib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/MetricsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,22 @@ public class MetricsTest
@Test
public void testMetricsConfiguration()
{
Metrics metrics = new Metrics("test", null);
Metrics metrics = new Metrics("test", null, true, true);
DimensionsAndCollector dimensionsAndCollector = metrics.getByName("query/time", "historical");
Assert.assertNotNull(dimensionsAndCollector);
String[] dimensions = dimensionsAndCollector.getDimensions();
Assert.assertEquals("dataSource", dimensions[0]);
Assert.assertEquals("type", dimensions[1]);
Assert.assertEquals("druid_service", dimensions[1]);
Assert.assertEquals("host_name", dimensions[2]);
Assert.assertEquals("type", dimensions[3]);
Assert.assertEquals(1000.0, dimensionsAndCollector.getConversionFactor(), 0.0);
Assert.assertTrue(dimensionsAndCollector.getCollector() instanceof Histogram);

DimensionsAndCollector d = metrics.getByName("segment/loadQueue/count", "historical");
Assert.assertNotNull(d);
String[] dims = d.getDimensions();
Assert.assertEquals("server", dims[0]);
Assert.assertEquals("druid_service", dims[0]);
Assert.assertEquals("host_name", dims[1]);
Assert.assertEquals("server", dims[2]);
}
}
54 changes: 39 additions & 15 deletions ...heus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,54 +38,78 @@
public class PrometheusEmitterTest
{
@Test
public void testEmitter()
public void testEmitterWithServiceLabel()
{
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null);
CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, true);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("server", "druid-data01.vpc.region")
.build("segment/loadQueue/count", 10)
.build(ImmutableMap.of("service", "historical"));
.build(ImmutableMap.of("service", "historical", "host", "druid.test.cn"));
Assert.assertEquals("historical", build.getService());
Assert.assertEquals("druid.test.cn", build.getHost());
Assert.assertFalse(build.getUserDims().isEmpty());
emitter.emit(build);
Double count = CollectorRegistry.defaultRegistry.getSampleValue(
"druid_segment_loadqueue_count", new String[]{"server"}, new String[]{"druid_data01_vpc_region"}
"druid_segment_loadqueue_count", new String[]{"druid_service", "server"}, new String[]{"historical", "druid_data01_vpc_region"}
);
Assert.assertEquals(10, count.intValue());
}

@Test
public void testEmitterWithServiceAndHostLabel()
{
CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, true, true);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("server", "druid-data01.vpc.region")
.build("segment/loadQueue/count", 10)
.build(ImmutableMap.of("service", "historical", "host", "druid.test.cn"));
Assert.assertEquals("historical", build.getService());
Assert.assertEquals("druid.test.cn", build.getHost());
Assert.assertFalse(build.getUserDims().isEmpty());
emitter.emit(build);
Double count = CollectorRegistry.defaultRegistry.getSampleValue(
"druid_segment_loadqueue_count", new String[]{"druid_service", "host_name", "server"}, new String[]{"historical", "druid.test.cn", "druid_data01_vpc_region"}
);
Assert.assertEquals(10, count.intValue());
}

@Test
public void testEmitterMetric()
{
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace", null, 0, "pushgateway");
CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace", null, 0, "pushgateway", true, true);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("dataSource", "test")
.setDimension("taskType", "index_parallel")
.build("task/run/time", 500)
.build(ImmutableMap.of("service", "overlord"));
.build(ImmutableMap.of("service", "overlord", "host", "druid.test.cn"));
emitter.emit(build);
double assertEpsilon = 0.0001;
Assert.assertEquals(0.0, CollectorRegistry.defaultRegistry.getSampleValue(
"namespace_task_run_time_bucket", new String[]{"dataSource", "taskType", "le"}, new String[]{"test", "index_parallel", "0.1"}
"namespace_task_run_time_bucket", new String[]{"dataSource", "druid_service", "host_name", "taskType", "le"}, new String[]{"test", "overlord", "druid.test.cn", "index_parallel", "0.1"}
), assertEpsilon);
Assert.assertEquals(1.0, CollectorRegistry.defaultRegistry.getSampleValue(
"namespace_task_run_time_bucket", new String[]{"dataSource", "taskType", "le"}, new String[]{"test", "index_parallel", "0.5"}
"namespace_task_run_time_bucket", new String[]{"dataSource", "druid_service", "host_name", "taskType", "le"}, new String[]{"test", "overlord", "druid.test.cn", "index_parallel", "0.5"}
), assertEpsilon);
}

@Test
public void testEmitterStart()
{
PrometheusEmitterConfig exportEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, "namespace1", null, 0, null);
PrometheusEmitterConfig exportEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, "namespace1", null, 0, null, true, true);
PrometheusEmitter exportEmitter = new PrometheusEmitter(exportEmitterConfig);
exportEmitter.start();
Assert.assertNotNull(exportEmitter.getServer());

PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace2", null, 0, "pushgateway");
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace2", null, 0, "pushgateway", true, true);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
Expand All @@ -94,7 +118,7 @@ public void testEmitterStart()
@Test
public void testEmitterPush() throws IOException
{
PrometheusEmitterConfig emitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace3", null, 0, "pushgateway");
PrometheusEmitterConfig emitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace3", null, 0, "pushgateway", true, true);

PushGateway mockPushGateway = mock(PushGateway.class);
mockPushGateway.push(anyObject(Collector.class), anyString(), anyObject(ImmutableMap.class));
Expand All @@ -105,21 +129,21 @@ public void testEmitterPush() throws IOException
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("task", "index_parallel")
.build("task/run/time", 500)
.build(ImmutableMap.of("service", "peon"));
.build(ImmutableMap.of("service", "peon", "host", "druid.test.cn"));
emitter.emit(build);
emitter.flush();
}

@Test
public void testEmitterConfigCreationWithNullAsAddress()
{
Assert.assertThrows(NullPointerException.class, () -> new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace4", null, 0, null));
Assert.assertThrows(NullPointerException.class, () -> new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace4", null, 0, null, true, true));
}

@Test
public void testEmitterStartWithHttpUrl()
{
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace4", null, 0, "http://pushgateway");
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace4", null, 0, "http://pushgateway", true, true);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
Expand All @@ -128,7 +152,7 @@ public void testEmitterStartWithHttpUrl()
@Test
public void testEmitterStartWithHttpsUrl()
{
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace5", null, 0, "https://pushgateway");
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace5", null, 0, "https://pushgateway", true, true);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
Expand Down