Exports GRPC call metrics and Feast resource metrics in Core (#345)

* Add dependency for prometheus client and servlet

* Add MonitoringInterceptor to Feast core service
To obtain request latency histogram for all RPC call in Prometheus

* Register servlet for Prometheus client

* Add custom Prometheus collector to export metrics about Feast resources

* Add JVMResourceCollector to collect JVM metrics and garbage collection

* Apply maven spotless

core/pom.xml

@@ -167,6 +167,16 @@
             <artifactId>hamcrest-library</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_servlet</artifactId>
+        </dependency>
+
         <!--testCompile 'com.jayway.jsonpath:json-path-assert:2.2.0'-->
         <dependency>
             <groupId>com.jayway.jsonpath</groupId>

core/src/main/java/feast/core/config/MonitoringConfig.java

@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.config;
+
+import feast.core.dao.FeatureSetRepository;
+import feast.core.dao.StoreRepository;
+import feast.core.metrics.collector.FeastResourceCollector;
+import feast.core.metrics.collector.JVMResourceCollector;
+import io.prometheus.client.exporter.MetricsServlet;
+import javax.servlet.http.HttpServlet;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.web.servlet.ServletRegistrationBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class MonitoringConfig {
+
+  private static final String PROMETHEUS_METRICS_PATH = "/metrics";
+
+  /**
+   * Add Prometheus exposition to an existing HTTP server using servlets.
+   *
+   * <p>https://github.com/prometheus/client_java/tree/b61dd232a504e20dad404a2bf3e2c0b8661c212a#http
+   *
+   * @return HTTP servlet for returning metrics data
+   */
+  @Bean
+  public ServletRegistrationBean<HttpServlet> metricsServlet() {
+    return new ServletRegistrationBean<>(new MetricsServlet(), PROMETHEUS_METRICS_PATH);
+  }
+
+  /**
+   * Register custom Prometheus collector that exports metrics about Feast Resources.
+   *
+   * <p>For example: total number of registered feature sets and stores.
+   *
+   * @param featureSetRepository {@link FeatureSetRepository}
+   * @param storeRepository {@link StoreRepository}
+   * @return {@link FeastResourceCollector}
+   */
+  @Bean
+  @Autowired
+  public FeastResourceCollector feastResourceCollector(
+      FeatureSetRepository featureSetRepository, StoreRepository storeRepository) {
+    FeastResourceCollector collector =
+        new FeastResourceCollector(featureSetRepository, storeRepository);
+    collector.register();
+    return collector;
+  }
+
+  /**
+   * Register custom Prometheus collector that exports metrics about JVM resource usage.
+   *
+   * @return @{link {@link JVMResourceCollector}}
+   */
+  @Bean
+  public JVMResourceCollector jvmResourceCollector() {
+    JVMResourceCollector jvmResourceCollector = new JVMResourceCollector();
+    jvmResourceCollector.register();
+    return jvmResourceCollector;
+  }
+}

core/src/main/java/feast/core/dao/FeatureSetRepository.java

@@ -24,6 +24,8 @@
 /** JPA repository supplying FeatureSet objects keyed by id. */
 public interface FeatureSetRepository extends JpaRepository<FeatureSet, String> {
 
+  long count();
+
   // Find feature set by name and version
   FeatureSet findFeatureSetByNameAndVersion(String name, Integer version);
 

core/src/main/java/feast/core/dao/StoreRepository.java

@@ -20,4 +20,6 @@
 import org.springframework.data.jpa.repository.JpaRepository;
 
 /** JPA repository supplying Store objects keyed by id. */
-public interface StoreRepository extends JpaRepository<Store, String> {}
+public interface StoreRepository extends JpaRepository<Store, String> {
+  long count();
+}

core/src/main/java/feast/core/grpc/CoreServiceImpl.java

@@ -38,6 +38,7 @@
 import feast.core.StoreProto.Store;
 import feast.core.StoreProto.Store.Subscription;
 import feast.core.exception.RetrievalException;
+import feast.core.grpc.interceptors.MonitoringInterceptor;
 import feast.core.service.JobCoordinatorService;
 import feast.core.service.SpecService;
 import io.grpc.stub.StreamObserver;
@@ -51,7 +52,7 @@
 
 /** Implementation of the feast core GRPC service. */
 @Slf4j
-@GRpcService
+@GRpcService(interceptors = {MonitoringInterceptor.class})
 public class CoreServiceImpl extends CoreServiceImplBase {
 
   private SpecService specService;

core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java

@@ -0,0 +1,56 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.grpc.interceptors;
+
+import feast.core.metrics.GrpcMetrics;
+import io.grpc.ForwardingServerCall.SimpleForwardingServerCall;
+import io.grpc.Metadata;
+import io.grpc.MethodDescriptor;
+import io.grpc.ServerCall;
+import io.grpc.ServerCall.Listener;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+import io.grpc.Status;
+
+/**
+ * MonitoringInterceptor intercepts a GRPC call to provide a request latency historgram metrics in
+ * the Prometheus client.
+ */
+public class MonitoringInterceptor implements ServerInterceptor {
+
+  @Override
+  public <ReqT, RespT> Listener<ReqT> interceptCall(
+      ServerCall<ReqT, RespT> call, Metadata headers, ServerCallHandler<ReqT, RespT> next) {
+
+    long startCallMillis = System.currentTimeMillis();
+    String fullMethodName = call.getMethodDescriptor().getFullMethodName();
+    String serviceName = MethodDescriptor.extractFullServiceName(fullMethodName);
+    String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1);
+
+    return next.startCall(
+        new SimpleForwardingServerCall<ReqT, RespT>(call) {
+          @Override
+          public void close(Status status, Metadata trailers) {
+            GrpcMetrics.requestLatency
+                .labels(serviceName, methodName, status.getCode().name())
+                .observe((System.currentTimeMillis() - startCallMillis) / 1000f);
+            super.close(status, trailers);
+          }
+        },
+        headers);
+  }
+}

core/src/main/java/feast/core/metrics/GrpcMetrics.java

@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.metrics;
+
+import io.prometheus.client.Histogram;
+
+public class GrpcMetrics {
+
+  public static final Histogram requestLatency =
+      Histogram.build()
+          .name("feast_core_request_latency_seconds")
+          .labelNames("service", "method", "status_code")
+          .help("Request latency in seconds")
+          .register();
+}

core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java

@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.metrics.collector;
+
+import feast.core.dao.FeatureSetRepository;
+import feast.core.dao.StoreRepository;
+import io.prometheus.client.Collector;
+import io.prometheus.client.GaugeMetricFamily;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * FeastResourceCollector exports metrics about Feast Resources.
+ *
+ * <p>For example: total number of registered feature sets and stores.
+ */
+public class FeastResourceCollector extends Collector {
+
+  private final FeatureSetRepository featureSetRepository;
+  private final StoreRepository storeRepository;
+
+  public FeastResourceCollector(
+      FeatureSetRepository featureSetRepository, StoreRepository storeRepository) {
+    this.featureSetRepository = featureSetRepository;
+    this.storeRepository = storeRepository;
+  }
+
+  @Override
+  public List<MetricFamilySamples> collect() {
+    List<MetricFamilySamples> samples = new ArrayList<>();
+    samples.add(
+        new GaugeMetricFamily(
+            "feast_core_feature_set_total",
+            "Total number of registered feature sets",
+            featureSetRepository.count()));
+    samples.add(
+        new GaugeMetricFamily(
+            "feast_core_store_total",
+            "Total number of registered stores",
+            storeRepository.count()));
+    return samples;
+  }
+}

core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java

@@ -0,0 +1,76 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.metrics.collector;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.GaugeMetricFamily;
+import io.prometheus.client.SummaryMetricFamily;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * JVMResourceCollector exports metrics about Java virtual machine memory and garbage collection.
+ */
+public class JVMResourceCollector extends Collector {
+
+  private final List<GarbageCollectorMXBean> garbageCollectors;
+  private final Runtime runtime;
+
+  public JVMResourceCollector() {
+    garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans();
+    runtime = Runtime.getRuntime();
+  }
+
+  @Override
+  public List<MetricFamilySamples> collect() {
+    List<MetricFamilySamples> samples = new ArrayList<>();
+
+    samples.add(
+        new GaugeMetricFamily(
+            "feast_core_max_memory_bytes",
+            "Max amount of memory the Java virtual machine will attempt to use",
+            runtime.maxMemory()));
+    samples.add(
+        new GaugeMetricFamily(
+            "feast_core_total_memory_bytes",
+            "Total amount of memory in the Java virtual machine",
+            runtime.totalMemory()));
+    samples.add(
+        new GaugeMetricFamily(
+            "feast_core_free_memory_bytes",
+            "Total amount of free memory in the Java virtual machine",
+            runtime.freeMemory()));
+
+    SummaryMetricFamily gcMetricFamily =
+        new SummaryMetricFamily(
+            "feast_core_gc_collection_seconds",
+            "Time spent in a given JVM garbage collector in seconds",
+            Collections.singletonList("gc"));
+    for (final GarbageCollectorMXBean gc : garbageCollectors) {
+      gcMetricFamily.addMetric(
+          Collections.singletonList(gc.getName()),
+          gc.getCollectionCount(),
+          gc.getCollectionTime() / MILLISECONDS_PER_SECOND);
+    }
+    samples.add(gcMetricFamily);
+
+    return samples;
+  }
+}

pom.xml

@@ -47,6 +47,7 @@
         <springBootVersion>2.0.9.RELEASE</springBootVersion>
         <org.apache.beam.version>2.16.0</org.apache.beam.version>
         <com.google.cloud.version>1.91.0</com.google.cloud.version>
+        <io.prometheus.version>0.8.0</io.prometheus.version>
 
         <byte-buddy.version>1.9.10</byte-buddy.version>
         <hamcrest.version>1.3</hamcrest.version>
@@ -170,6 +171,18 @@
                 <version>3.0.2</version>
             </dependency>
 
+            <!-- Prometheus Client -->
+            <dependency>
+                <groupId>io.prometheus</groupId>
+                <artifactId>simpleclient</artifactId>
+                <version>${io.prometheus.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.prometheus</groupId>
+                <artifactId>simpleclient_servlet</artifactId>
+                <version>${io.prometheus.version}</version>
+            </dependency>
+
             <!-- Other Stuff -->
             <dependency>
               <groupId>com.datadoghq</groupId>