[ISSUE apache#340]merge otel metric branch 2 develop (apache#494)

* [ISSUE apache#340]Integrate With OpenTelemetry for metrics in EventMesh (apache#467)

* The Chinese in the notes is translated into English.

* Translation improvement

* Translation improvement

* fix Chinese annotation on runtime module

* Export metrics data with open telemetry and use Prometheus for visual observation

* Export metrics data with open telemetry and use Prometheus for visual observation

* Export metrics data with open telemetry and use Prometheus for visual observation

* Export metrics data with open telemetry and use Prometheus for visual observation

* Improper modification

* Improper modification

* improve

* improve

* improve

* tcp metrics export

* improve

* improve

* merge from otel branch

* check lisence

Co-authored-by: ZePeng Chen <[email protected]>

build.gradle

@@ -364,6 +364,13 @@ subprojects {
             dependency "io.dropwizard.metrics:metrics-annotation:4.1.0"
             dependency "io.dropwizard.metrics:metrics-json:4.1.0"
 
+            dependency 'io.opentelemetry:opentelemetry-api:1.3.0'
+            dependency 'io.opentelemetry:opentelemetry-sdk:1.3.0'
+            dependency 'io.opentelemetry:opentelemetry-sdk-metrics:1.3.0-alpha'
+            dependency 'io.opentelemetry:opentelemetry-exporter-prometheus:1.3.0-alpha'
+            dependency 'io.prometheus:simpleclient:0.8.1'
+            dependency 'io.prometheus:simpleclient_httpserver:0.8.1'
+
             dependency "io.openmessaging:openmessaging-api:2.2.1-pubsub"
 
             dependency "com.h3xstream.findsecbugs:findsecbugs-plugin:1.11.0"

docs/en/features/eventmesh-metrics-export-design.md

@@ -0,0 +1,47 @@
+# EventMesh Metrics (OpenTelemetry+Prometheus)
+
+## Introduction
+
+[EventMesh(incubating)](https://github.com/apache/incubator-eventmesh) is a dynamic cloud-native eventing infrastructure.
+
+## An overview of OpenTelemetry
+
+OpenTelemetry is a collection of tools, APIs, and SDKs. You can use it to instrument, generate, collect, and export telemetry data (metrics, logs, and traces) for analysis in order to understand your software's performance and behavior.
+
+## An overview of  Prometheus
+
+Power your metrics and alerting with a leading open-source monitoring solution.
+
+- Dimensional data
+- Powerful queries
+- Great visualization
+- Efficient storage
+- Simple operation 
+- Precise alerting
+- Many client libraries
+- Many integrations
+
+## Requirements
+
+### Functional Requirements
+
+| Requirement ID | Requirement Description                                      | Comments      |
+| :------------- | ------------------------------------------------------------ | ------------- |
+| F-1            | EventMesh users should be able to observe HTTP metrics from Prometheus | Functionality |
+| F-2            | EventMesh users should be able to observe TCP metrics from Prometheus | Functionality |
+
+## Design Details
+
+use the meter instrument provided by OpenTelemetry to observe the metrics exist in EventMesh then export to Prometheus.
+
+1、Initialize a meter instrument
+
+2、set the Prometheus server
+
+3、different metrics observer built
+
+## Appendix
+
+#### References
+
+https://github.com/open-telemetry/docs-cn/blob/main/QUICKSTART.md#%E5%88%9B%E5%BB%BA%E5%9F%BA%E7%A1%80Span

eventmesh-common/src/main/java/org/apache/eventmesh/common/config/CommonConfiguration.java

@@ -30,6 +30,7 @@ public class CommonConfiguration {
     public String sysID = "5477";
     public String eventMeshConnectorPluginType = "rocketmq";
     public String eventMeshSecurityPluginType = "security";
+    public int eventMeshPrometheusPort = 19090;
 
     public String namesrvAddr = "";
     public Integer eventMeshRegisterIntervalInMills = 10 * 1000;
@@ -65,6 +66,11 @@ public void init() {
             Preconditions.checkState(StringUtils.isNotEmpty(eventMeshIDCStr), String.format("%s error", ConfKeys.KEYS_EVENTMESH_IDC));
             eventMeshIDC = StringUtils.deleteWhitespace(eventMeshIDCStr);
 
+            String eventMeshPrometheusPortStr = configurationWrapper.getProp(ConfKeys.KEY_EVENTMESH_METRICS_PROMETHEUS_PORT);
+            if (StringUtils.isNotEmpty(eventMeshPrometheusPortStr)) {
+                eventMeshPrometheusPort = Integer.valueOf(StringUtils.deleteWhitespace(eventMeshPrometheusPortStr));
+            }
+
             eventMeshServerIp = configurationWrapper.getProp(ConfKeys.KEYS_EVENTMESH_SERVER_HOST_IP);
             if (StringUtils.isBlank(eventMeshServerIp)) {
                 eventMeshServerIp = IPUtil.getLocalAddress();
@@ -105,5 +111,7 @@ static class ConfKeys {
         public static String KEYS_EVENTMESH_SECURITY_ENABLED = "eventMesh.server.security.enabled";
 
         public static String KEYS_ENENTMESH_SECURITY_PLUGIN_TYPE = "eventMesh.security.plugin.type";
+
+        public static String KEY_EVENTMESH_METRICS_PROMETHEUS_PORT = "eventMesh.metrics.prometheus.port";
     }
 }

eventmesh-runtime/build.gradle

@@ -16,6 +16,13 @@
  */
 
 dependencies {
+    api 'io.opentelemetry:opentelemetry-api'
+    api 'io.opentelemetry:opentelemetry-sdk'
+    api 'io.opentelemetry:opentelemetry-sdk-metrics'
+    api 'io.opentelemetry:opentelemetry-exporter-prometheus'
+    api 'io.prometheus:simpleclient'
+    api 'io.prometheus:simpleclient_httpserver'
+
     implementation project(":eventmesh-connector-plugin:eventmesh-connector-api")
     implementation project(":eventmesh-security-plugin:eventmesh-security-api")
     implementation project(":eventmesh-security-plugin:eventmesh-security-acl")

eventmesh-runtime/conf/eventmesh.properties

@@ -60,4 +60,7 @@ eventMesh.connector.plugin.type=rocketmq
 
 #security plugin
 eventMesh.server.security.enabled=false
-eventMesh.security.plugin.type=security
+eventMesh.security.plugin.type=security
+
+#prometheusPort
+eventMesh.metrics.prometheus.port=19090

eventmesh-runtime/conf/prometheus.yml

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: []
+    scheme: http
+    timeout: 10s
+    api_version: v1
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: EventMesh_HTTP_export_test
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - 127.0.0.1:19090

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfiguration.java

@@ -226,5 +226,6 @@ static class ConfKeys {
         public static String KEY_EVENTMESH_CONSUMER_ENABLED = "eventMesh.server.consumer.enabled";
 
         public static String KEY_EVENTMESH_HTTPS_ENABLED = "eventMesh.server.useTls.enabled";
+
     }
 }

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HTTPMetricsServer.java

@@ -26,6 +26,7 @@
 import com.codahale.metrics.MetricRegistry;
 
 import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer;
+import org.apache.eventmesh.runtime.metrics.openTelemetry.OpenTelemetryHTTPMetricsExporter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,6 +44,8 @@ public class HTTPMetricsServer {
 
     public GroupMetrics groupMetrics;
 
+    public OpenTelemetryHTTPMetricsExporter openTelemetryHTTPMetricsExporter;
+
     private Logger httpLogger = LoggerFactory.getLogger("httpMonitor");
 
     private Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -56,10 +59,14 @@ public void init() throws Exception {
         topicMetrics = new TopicMetrics(this.eventMeshHTTPServer, this.metricRegistry);
         groupMetrics = new GroupMetrics(this.eventMeshHTTPServer, this.metricRegistry);
         healthMetrics = new HealthMetrics(this.eventMeshHTTPServer, this.metricRegistry);
+
+        openTelemetryHTTPMetricsExporter = new OpenTelemetryHTTPMetricsExporter(this,this.eventMeshHTTPServer.getEventMeshHttpConfiguration());
+
         logger.info("HTTPMetricsServer inited......");
     }
 
     public void start() throws Exception {
+        openTelemetryHTTPMetricsExporter.start();
         metricsSchedule.scheduleAtFixedRate(new Runnable() {
             @Override
             public void run() {
@@ -90,6 +97,7 @@ public void run() {
 
     public void shutdown() throws Exception {
         metricsSchedule.shutdown();
+        openTelemetryHTTPMetricsExporter.shutdown();
         logger.info("HTTPMetricsServer shutdown......");
     }
 
@@ -162,6 +170,21 @@ private void logPrintServerMetrics() {
         summaryMetrics.send2MQStatInfoClear();
     }
 
+    public int getBatchMsgQ(){
+        return eventMeshHTTPServer.getBatchMsgExecutor().getQueue().size();
+    }
+
+    public int getSendMsgQ(){
+        return eventMeshHTTPServer.getSendMsgExecutor().getQueue().size();
+    }
+
+    public int getPushMsgQ(){
+        return eventMeshHTTPServer.getPushMsgExecutor().getQueue().size();
+    }
+
+    public int getHttpRetryQ(){
+        return eventMeshHTTPServer.getHttpRetryer().size();
+    }
 
     public HealthMetrics getHealthMetrics() {
         return healthMetrics;

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/openTelemetry/OpenTelemetryExporterConfiguration.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.runtime.metrics.openTelemetry;
+
+import io.opentelemetry.api.metrics.MeterProvider;
+import io.opentelemetry.exporter.prometheus.PrometheusCollector;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.prometheus.client.exporter.HTTPServer;
+import org.apache.eventmesh.common.config.CommonConfiguration;
+import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration;
+
+import java.io.IOException;
+
+//ues openTelemetry to export metrics data
+public class OpenTelemetryExporterConfiguration {
+
+    private static HTTPServer server;//Prometheus server
+
+    static int prometheusPort;//the endpoint to export metrics
+
+    static MeterProvider meterProvider;
+    /**
+     * Initializes the Meter SDK and configures the prometheus collector with all default settings.
+     *
+     * @return A MeterProvider for use in instrumentation.
+     */
+    public MeterProvider initializeOpenTelemetry(CommonConfiguration configuration) {
+        if (server!=null){//the sever already start
+            return meterProvider;
+        }
+
+        prometheusPort = configuration.eventMeshPrometheusPort;
+        SdkMeterProvider sdkMeterProvider = SdkMeterProvider.builder().buildAndRegisterGlobal();
+        PrometheusCollector.builder().setMetricProducer(sdkMeterProvider).buildAndRegister();
+        this.meterProvider = sdkMeterProvider;
+        try {
+            server = new HTTPServer(prometheusPort,true);//Use the daemon thread to start an HTTP server to serve the default Prometheus registry.
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return meterProvider;
+    }
+
+    public static MeterProvider getMeterProvider(){//for tcp or http to get the initialized meterProvider
+        return meterProvider;
+    }
+
+    public void shutdownPrometheusEndpoint() {
+        if (server==null)
+            return;
+        server.stop();
+    }
+}