diff --git a/core/pom.xml b/core/pom.xml index e9dfee3311..f6e4909260 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -167,6 +167,16 @@ hamcrest-library + + io.prometheus + simpleclient + + + + io.prometheus + simpleclient_servlet + + com.jayway.jsonpath diff --git a/core/src/main/java/feast/core/config/MonitoringConfig.java b/core/src/main/java/feast/core/config/MonitoringConfig.java new file mode 100644 index 0000000000..fd20bed1ee --- /dev/null +++ b/core/src/main/java/feast/core/config/MonitoringConfig.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.config; + +import feast.core.dao.FeatureSetRepository; +import feast.core.dao.StoreRepository; +import feast.core.metrics.collector.FeastResourceCollector; +import feast.core.metrics.collector.JVMResourceCollector; +import io.prometheus.client.exporter.MetricsServlet; +import javax.servlet.http.HttpServlet; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.web.servlet.ServletRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MonitoringConfig { + + private static final String PROMETHEUS_METRICS_PATH = "/metrics"; + + /** + * Add Prometheus exposition to an existing HTTP server using servlets. + * + *

https://github.com/prometheus/client_java/tree/b61dd232a504e20dad404a2bf3e2c0b8661c212a#http + * + * @return HTTP servlet for returning metrics data + */ + @Bean + public ServletRegistrationBean metricsServlet() { + return new ServletRegistrationBean<>(new MetricsServlet(), PROMETHEUS_METRICS_PATH); + } + + /** + * Register custom Prometheus collector that exports metrics about Feast Resources. + * + *

For example: total number of registered feature sets and stores. + * + * @param featureSetRepository {@link FeatureSetRepository} + * @param storeRepository {@link StoreRepository} + * @return {@link FeastResourceCollector} + */ + @Bean + @Autowired + public FeastResourceCollector feastResourceCollector( + FeatureSetRepository featureSetRepository, StoreRepository storeRepository) { + FeastResourceCollector collector = + new FeastResourceCollector(featureSetRepository, storeRepository); + collector.register(); + return collector; + } + + /** + * Register custom Prometheus collector that exports metrics about JVM resource usage. + * + * @return @{link {@link JVMResourceCollector}} + */ + @Bean + public JVMResourceCollector jvmResourceCollector() { + JVMResourceCollector jvmResourceCollector = new JVMResourceCollector(); + jvmResourceCollector.register(); + return jvmResourceCollector; + } +} diff --git a/core/src/main/java/feast/core/dao/FeatureSetRepository.java b/core/src/main/java/feast/core/dao/FeatureSetRepository.java index e8d37424da..9f23cb4aee 100644 --- a/core/src/main/java/feast/core/dao/FeatureSetRepository.java +++ b/core/src/main/java/feast/core/dao/FeatureSetRepository.java @@ -24,6 +24,8 @@ /** JPA repository supplying FeatureSet objects keyed by id. */ public interface FeatureSetRepository extends JpaRepository { + long count(); + // Find feature set by name and version FeatureSet findFeatureSetByNameAndVersion(String name, Integer version); diff --git a/core/src/main/java/feast/core/dao/StoreRepository.java b/core/src/main/java/feast/core/dao/StoreRepository.java index a0015b9932..2cc0070471 100644 --- a/core/src/main/java/feast/core/dao/StoreRepository.java +++ b/core/src/main/java/feast/core/dao/StoreRepository.java @@ -20,4 +20,6 @@ import org.springframework.data.jpa.repository.JpaRepository; /** JPA repository supplying Store objects keyed by id. */ -public interface StoreRepository extends JpaRepository {} +public interface StoreRepository extends JpaRepository { + long count(); +} diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index a057789df7..eba4d99c46 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -38,6 +38,7 @@ import feast.core.StoreProto.Store; import feast.core.StoreProto.Store.Subscription; import feast.core.exception.RetrievalException; +import feast.core.grpc.interceptors.MonitoringInterceptor; import feast.core.service.JobCoordinatorService; import feast.core.service.SpecService; import io.grpc.stub.StreamObserver; @@ -51,7 +52,7 @@ /** Implementation of the feast core GRPC service. */ @Slf4j -@GRpcService +@GRpcService(interceptors = {MonitoringInterceptor.class}) public class CoreServiceImpl extends CoreServiceImplBase { private SpecService specService; diff --git a/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java b/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java new file mode 100644 index 0000000000..870675594a --- /dev/null +++ b/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.grpc.interceptors; + +import feast.core.metrics.GrpcMetrics; +import io.grpc.ForwardingServerCall.SimpleForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.ServerCall; +import io.grpc.ServerCall.Listener; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * MonitoringInterceptor intercepts a GRPC call to provide a request latency historgram metrics in + * the Prometheus client. + */ +public class MonitoringInterceptor implements ServerInterceptor { + + @Override + public Listener interceptCall( + ServerCall call, Metadata headers, ServerCallHandler next) { + + long startCallMillis = System.currentTimeMillis(); + String fullMethodName = call.getMethodDescriptor().getFullMethodName(); + String serviceName = MethodDescriptor.extractFullServiceName(fullMethodName); + String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1); + + return next.startCall( + new SimpleForwardingServerCall(call) { + @Override + public void close(Status status, Metadata trailers) { + GrpcMetrics.requestLatency + .labels(serviceName, methodName, status.getCode().name()) + .observe((System.currentTimeMillis() - startCallMillis) / 1000f); + super.close(status, trailers); + } + }, + headers); + } +} diff --git a/core/src/main/java/feast/core/metrics/GrpcMetrics.java b/core/src/main/java/feast/core/metrics/GrpcMetrics.java new file mode 100644 index 0000000000..7c20e63250 --- /dev/null +++ b/core/src/main/java/feast/core/metrics/GrpcMetrics.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.metrics; + +import io.prometheus.client.Histogram; + +public class GrpcMetrics { + + public static final Histogram requestLatency = + Histogram.build() + .name("feast_core_request_latency_seconds") + .labelNames("service", "method", "status_code") + .help("Request latency in seconds") + .register(); +} diff --git a/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java b/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java new file mode 100644 index 0000000000..b79ea5a3c3 --- /dev/null +++ b/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.metrics.collector; + +import feast.core.dao.FeatureSetRepository; +import feast.core.dao.StoreRepository; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import java.util.ArrayList; +import java.util.List; + +/** + * FeastResourceCollector exports metrics about Feast Resources. + * + *

For example: total number of registered feature sets and stores. + */ +public class FeastResourceCollector extends Collector { + + private final FeatureSetRepository featureSetRepository; + private final StoreRepository storeRepository; + + public FeastResourceCollector( + FeatureSetRepository featureSetRepository, StoreRepository storeRepository) { + this.featureSetRepository = featureSetRepository; + this.storeRepository = storeRepository; + } + + @Override + public List collect() { + List samples = new ArrayList<>(); + samples.add( + new GaugeMetricFamily( + "feast_core_feature_set_total", + "Total number of registered feature sets", + featureSetRepository.count())); + samples.add( + new GaugeMetricFamily( + "feast_core_store_total", + "Total number of registered stores", + storeRepository.count())); + return samples; + } +} diff --git a/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java b/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java new file mode 100644 index 0000000000..8602f6c249 --- /dev/null +++ b/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.metrics.collector; + +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import io.prometheus.client.SummaryMetricFamily; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * JVMResourceCollector exports metrics about Java virtual machine memory and garbage collection. + */ +public class JVMResourceCollector extends Collector { + + private final List garbageCollectors; + private final Runtime runtime; + + public JVMResourceCollector() { + garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans(); + runtime = Runtime.getRuntime(); + } + + @Override + public List collect() { + List samples = new ArrayList<>(); + + samples.add( + new GaugeMetricFamily( + "feast_core_max_memory_bytes", + "Max amount of memory the Java virtual machine will attempt to use", + runtime.maxMemory())); + samples.add( + new GaugeMetricFamily( + "feast_core_total_memory_bytes", + "Total amount of memory in the Java virtual machine", + runtime.totalMemory())); + samples.add( + new GaugeMetricFamily( + "feast_core_free_memory_bytes", + "Total amount of free memory in the Java virtual machine", + runtime.freeMemory())); + + SummaryMetricFamily gcMetricFamily = + new SummaryMetricFamily( + "feast_core_gc_collection_seconds", + "Time spent in a given JVM garbage collector in seconds", + Collections.singletonList("gc")); + for (final GarbageCollectorMXBean gc : garbageCollectors) { + gcMetricFamily.addMetric( + Collections.singletonList(gc.getName()), + gc.getCollectionCount(), + gc.getCollectionTime() / MILLISECONDS_PER_SECOND); + } + samples.add(gcMetricFamily); + + return samples; + } +} diff --git a/pom.xml b/pom.xml index f8524afc88..edf2a0244e 100644 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,7 @@ 2.0.9.RELEASE 2.16.0 1.91.0 + 0.8.0 1.9.10 1.3 @@ -170,6 +171,18 @@ 3.0.2 + + + io.prometheus + simpleclient + ${io.prometheus.version} + + + io.prometheus + simpleclient_servlet + ${io.prometheus.version} + + com.datadoghq