Skip to content

Commit

Permalink
Merge pull request #37123 from iocanel/generate-servicemonitor
Browse files Browse the repository at this point in the history
Generate prometheus ServiceMonitor CR
  • Loading branch information
iocanel authored Nov 16, 2023
2 parents 20f0440 + cc38183 commit 2c45aa6
Show file tree
Hide file tree
Showing 16 changed files with 318 additions and 40 deletions.
8 changes: 7 additions & 1 deletion bom/application/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@
<kotlin.coroutine.version>1.7.3</kotlin.coroutine.version>
<azure.toolkit-lib.version>0.27.0</azure.toolkit-lib.version>
<kotlin-serialization.version>1.6.0</kotlin-serialization.version>
<dekorate.version>4.0.3</dekorate.version> <!-- Please check with Java Operator SDK team before updating -->
<dekorate.version>4.1.0</dekorate.version> <!-- Please check with Java Operator SDK team before updating -->
<maven-invoker.version>3.2.0</maven-invoker.version>
<awaitility.version>4.2.0</awaitility.version>
<jboss-logmanager.version>3.0.2.Final</jboss-logmanager.version>
Expand Down Expand Up @@ -3179,6 +3179,12 @@
<version>${dekorate.version}</version>
<classifier>noapt</classifier>
</dependency>
<dependency>
<groupId>io.dekorate</groupId>
<artifactId>prometheus-annotations</artifactId>
<version>${dekorate.version}</version>
<classifier>noapt</classifier>
</dependency>
<dependency>
<groupId>io.dekorate</groupId>
<artifactId>s2i-annotations</artifactId>
Expand Down
20 changes: 20 additions & 0 deletions extensions/kubernetes/vanilla/deployment/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,26 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.dekorate</groupId>
<artifactId>prometheus-annotations</artifactId>
<classifier>noapt</classifier>
<exclusions>
<exclusion>
<groupId>io.sundr</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun</groupId>
<artifactId>tools</artifactId>
</exclusion>
<exclusion>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-client</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-internal</artifactId>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package io.quarkus.kubernetes.deployment;

import io.dekorate.kubernetes.decorator.ResourceProvidingDecorator;
import io.dekorate.prometheus.model.ServiceMonitorBuilder;
import io.fabric8.kubernetes.api.model.KubernetesListBuilder;
import io.fabric8.kubernetes.api.model.ObjectMeta;

public class AddServiceMonitorResourceDecorator extends ResourceProvidingDecorator<KubernetesListBuilder> {

private final String scheme;
private final String targetPort;
private final String path;
private final int interval;
private final boolean honorLabels;

public AddServiceMonitorResourceDecorator(String scheme, String targetPort, String path, int interval,
boolean honorLabels) {
this.scheme = scheme;
this.targetPort = targetPort;
this.path = path;
this.interval = interval;
this.honorLabels = honorLabels;
}

@Override
public void visit(KubernetesListBuilder list) {
ObjectMeta meta = getMandatoryDeploymentMetadata(list, ANY);
list.addToItems(new ServiceMonitorBuilder()
.withNewMetadata()
.withName(meta.getName())
.withLabels(meta.getLabels())
.endMetadata()
.withNewSpec()
.withNewSelector()
.addToMatchLabels(meta.getLabels())
.endSelector()
.addNewEndpoint()
.withScheme(scheme)
.withNewTargetPort(targetPort)
.withPath(path)
.withInterval(interval + "s")
.withHonorLabels(honorLabels)
.endEndpoint()
.endSpec());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -996,12 +996,21 @@ private static List<DecoratorBuildItem> createAnnotationDecorators(Optional<Proj
now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd - HH:mm:ss Z")), new String[0]))));
}

if (config.getPrometheusConfig().annotations) {
// Add metrics annotations
metricsConfiguration.ifPresent(m -> {
String path = m.metricsEndpoint();
String prefix = config.getPrometheusConfig().prefix;
if (port.isPresent() && path != null) {
// Add metrics annotations
metricsConfiguration.ifPresent(m -> {
String path = m.metricsEndpoint();
String prefix = config.getPrometheusConfig().prefix;
if (port.isPresent() && path != null) {
if (config.getPrometheusConfig().generateServiceMonitor) {
result.add(new DecoratorBuildItem(target, new AddServiceMonitorResourceDecorator(
config.getPrometheusConfig().scheme.orElse("http"),
config.getPrometheusConfig().port.orElse(String.valueOf(port.get().getContainerPort())),
config.getPrometheusConfig().path.orElse(path),
10,
true)));
}

if (config.getPrometheusConfig().annotations) {
result.add(new DecoratorBuildItem(target, new AddAnnotationDecorator(name,
config.getPrometheusConfig().scrape.orElse(prefix + "/scrape"), "true",
PROMETHEUS_ANNOTATION_TARGETS)));
Expand All @@ -1014,8 +1023,8 @@ private static List<DecoratorBuildItem> createAnnotationDecorators(Optional<Proj
config.getPrometheusConfig().scheme.orElse(prefix + "/scheme"), "http",
PROMETHEUS_ANNOTATION_TARGETS)));
}
});
}
}
});

//Add metrics annotations
return result;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,16 @@ public class PrometheusConfig {
@ConfigItem(defaultValue = "true")
boolean annotations;

/**
* When true (the default), emit a set of annotations to identify
* services that should be scraped by prometheus for metrics.
*
* In configurations that use the Prometheus operator with ServiceMonitor,
* annotations may not be necessary.
*/
@ConfigItem(defaultValue = "true")
boolean generateServiceMonitor;

/**
* Define the annotation prefix used for scrape values, this value will be used
* as the base for other annotation name defaults. Altering the base for generated
Expand Down
11 changes: 11 additions & 0 deletions integration-tests/kubernetes/quarkus-standard-way/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,17 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.dekorate</groupId>
<artifactId>prometheus-annotations</artifactId>
<classifier>noapt</classifier>
<exclusions>
<exclusion>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-client</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-client</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

import io.dekorate.prometheus.model.Endpoint;
import io.dekorate.prometheus.model.ServiceMonitor;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.quarkus.builder.Version;
Expand All @@ -30,7 +32,10 @@ public class KubernetesWithMetricsCustomAbsoluteTest {
.setApplicationVersion("0.1-SNAPSHOT")
.setRun(true)
.setLogFileName("k8s.log")
.withConfigurationResource("kubernetes-with-metrics-custom-absolute.properties")
.overrideConfigKey("quarkus.http.port", "9090")
.overrideConfigKey("quarkus.micrometer.export.prometheus.path", "/absolute-metrics")
.overrideConfigKey("quarkus.kubernetes.prometheus.prefix", "example.io")
.overrideConfigKey("quarkus.kubernetes.prometheus.scrape", "example.io/should_be_scraped")
.setForcedDependencies(List.of(
Dependency.of("io.quarkus", "quarkus-micrometer-registry-prometheus", Version.getVersion())));

Expand Down Expand Up @@ -76,6 +81,22 @@ public void assertGeneratedResources() throws IOException {
});
});
});

assertThat(kubernetesList).filteredOn(i -> i.getKind().equals("ServiceMonitor")).singleElement()
.isInstanceOfSatisfying(ServiceMonitor.class, s -> {
assertThat(s.getMetadata()).satisfies(m -> {
assertThat(m.getName()).isEqualTo("metrics");
});

assertThat(s.getSpec()).satisfies(spec -> {
assertThat(spec.getEndpoints()).hasSize(1);
assertThat(spec.getEndpoints().get(0)).isInstanceOfSatisfying(Endpoint.class, e -> {
assertThat(e.getScheme()).isEqualTo("http");
assertThat(e.getTargetPort().getStrVal()).isEqualTo("9090");
assertThat(e.getPath()).isEqualTo("/absolute-metrics");
});
});
});
});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

import io.dekorate.prometheus.model.Endpoint;
import io.dekorate.prometheus.model.ServiceMonitor;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.quarkus.builder.Version;
Expand All @@ -30,8 +32,12 @@ public class KubernetesWithMetricsCustomRelativeTest {
.setApplicationVersion("0.1-SNAPSHOT")
.setRun(true)
.setLogFileName("k8s.log")
.withConfigurationResource("kubernetes-with-metrics-custom-relative.properties")
.setForcedDependencies(List.of(Dependency.of("io.quarkus", "quarkus-smallrye-metrics", Version.getVersion())));
.overrideConfigKey("quarkus.http.port", "9090")
.overrideConfigKey("quarkus.micrometer.export.prometheus.path", "met")
.overrideConfigKey("quarkus.kubernetes.prometheus.prefix", "example.io")
.overrideConfigKey("quarkus.kubernetes.prometheus.scrape", "example.io/should_be_scraped")
.setForcedDependencies(List.of(
Dependency.of("io.quarkus", "quarkus-micrometer-registry-prometheus", Version.getVersion())));

@ProdBuildResults
private ProdModeTestResults prodModeTestResults;
Expand Down Expand Up @@ -76,6 +82,22 @@ public void assertGeneratedResources() throws IOException {
});
});
});

assertThat(kubernetesList).filteredOn(i -> i.getKind().equals("ServiceMonitor")).singleElement()
.isInstanceOfSatisfying(ServiceMonitor.class, s -> {
assertThat(s.getMetadata()).satisfies(m -> {
assertThat(m.getName()).isEqualTo("metrics");
});

assertThat(s.getSpec()).satisfies(spec -> {
assertThat(spec.getEndpoints()).hasSize(1);
assertThat(spec.getEndpoints().get(0)).isInstanceOfSatisfying(Endpoint.class, e -> {
assertThat(e.getScheme()).isEqualTo("http");
assertThat(e.getTargetPort().getStrVal()).isEqualTo("9090");
assertThat(e.getPath()).isEqualTo("/q/met");
});
});
});
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

import io.dekorate.prometheus.model.Endpoint;
import io.dekorate.prometheus.model.ServiceMonitor;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.quarkus.builder.Version;
Expand All @@ -30,7 +32,10 @@ public class KubernetesWithMetricsNoAnnotationsTest {
.setApplicationVersion("0.1-SNAPSHOT")
.setRun(true)
.setLogFileName("k8s.log")
.withConfigurationResource("kubernetes-with-metrics-no-annotations.properties")
.overrideConfigKey("quarkus.http.port", "9090")
.overrideConfigKey("quarkus.smallrye-metrics.path", "/met")
.overrideConfigKey("quarkus.kubernetes.prometheus.annotations", "false")
.overrideConfigKey("quarkus.kubernetes.prometheus.prefix", "example.io")
.setForcedDependencies(List.of(
Dependency.of("io.quarkus", "quarkus-micrometer-registry-prometheus", Version.getVersion())));

Expand Down Expand Up @@ -76,6 +81,22 @@ public void assertGeneratedResources() throws IOException {
});
});
});

assertThat(kubernetesList).filteredOn(i -> i.getKind().equals("ServiceMonitor")).singleElement()
.isInstanceOfSatisfying(ServiceMonitor.class, s -> {
assertThat(s.getMetadata()).satisfies(m -> {
assertThat(m.getName()).isEqualTo("metrics");
});

assertThat(s.getSpec()).satisfies(spec -> {
assertThat(spec.getEndpoints()).hasSize(1);
assertThat(spec.getEndpoints().get(0)).isInstanceOfSatisfying(Endpoint.class, e -> {
assertThat(e.getScheme()).isEqualTo("http");
assertThat(e.getTargetPort().getStrVal()).isEqualTo("9090");
assertThat(e.getPath()).isEqualTo("/q/metrics");
});
});
});
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package io.quarkus.it.kubernetes;

import static io.restassured.RestAssured.given;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.entry;
import static org.hamcrest.Matchers.is;

import java.io.IOException;
import java.nio.file.Path;
import java.util.List;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.quarkus.builder.Version;
import io.quarkus.maven.dependency.Dependency;
import io.quarkus.test.LogFile;
import io.quarkus.test.ProdBuildResults;
import io.quarkus.test.ProdModeTestResults;
import io.quarkus.test.QuarkusProdModeTest;

public class KubernetesWithMetricsNoServiceMonitor {

@RegisterExtension
static final QuarkusProdModeTest config = new QuarkusProdModeTest()
.withApplicationRoot((jar) -> jar.addClasses(GreetingResource.class))
.setApplicationName("metrics")
.setApplicationVersion("0.1-SNAPSHOT")
.setRun(true)
.setLogFileName("k8s.log")
.overrideConfigKey("quarkus.http.port", "9090")
.overrideConfigKey("quarkus.smallrye-metrics.path", "/met")
.overrideConfigKey("quarkus.kubernetes.prometheus.generate-service-monitor", "false")
.overrideConfigKey("quarkus.kubernetes.prometheus.prefix", "example.io")
.setForcedDependencies(List.of(
Dependency.of("io.quarkus", "quarkus-micrometer-registry-prometheus", Version.getVersion())));

@ProdBuildResults
private ProdModeTestResults prodModeTestResults;

@LogFile
private Path logfile;

@Test
public void assertApplicationRuns() {
assertThat(logfile).isRegularFile().hasFileName("k8s.log");
TestUtil.assertLogFileContents(logfile, "kubernetes", "metrics");

given()
.when().get("/greeting")
.then()
.statusCode(200)
.body(is("hello"));
}

@Test
public void assertGeneratedResources() throws IOException {
final Path kubernetesDir = prodModeTestResults.getBuildDir().resolve("kubernetes");
assertThat(kubernetesDir)
.isDirectoryContaining(p -> p.getFileName().endsWith("kubernetes.json"))
.isDirectoryContaining(p -> p.getFileName().endsWith("kubernetes.yml"));
List<HasMetadata> kubernetesList = DeserializationUtil
.deserializeAsList(kubernetesDir.resolve("kubernetes.yml"));
assertThat(kubernetesList.get(0)).isInstanceOfSatisfying(Deployment.class, d -> {
assertThat(d.getMetadata()).satisfies(m -> {
assertThat(m.getName()).isEqualTo("metrics");
});

assertThat(d.getSpec()).satisfies(deploymentSpec -> {
assertThat(deploymentSpec.getTemplate()).satisfies(t -> {
assertThat(t.getMetadata()).satisfies(meta -> {
// Annotations should not have been created in this configuration.
assertThat(meta.getAnnotations()).contains(
entry("prometheus.io/scrape", "true"),
entry("prometheus.io/path", "/met"),
entry("prometheus.io/port", "9090"),
entry("prometheus.io/scheme", "http"));
});
});
});
});

assertThat(kubernetesList).filteredOn(i -> i.getKind().equals("ServiceMonitor")).isEmpty();
}
}
Loading

0 comments on commit 2c45aa6

Please sign in to comment.