Export JVM metrics from the javaagent using micrometer

bootstrap/build.gradle

@@ -7,5 +7,8 @@ compileJava {
 }
 
 dependencies {
-    implementation "org.slf4j:slf4j-api:1.7.30"
+    // slf4j is included in the otel javaagent, no need to add it here too
+    compileOnly "org.slf4j:slf4j-api:1.7.30"
+    // add micrometer to the bootstrap classloader so that it's available in instrumentations
+    implementation "io.micrometer:micrometer-core:${versions.micrometer}"
 }

bootstrap/src/main/java/com/splunk/opentelemetry/javaagent/bootstrap/GlobalMetricsTags.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Splunk Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.splunk.opentelemetry.javaagent.bootstrap;
+
+import io.micrometer.core.instrument.Tag;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class holds the list of tags that should be added to all meters registered by the javaagent.
+ * It needs to be loaded by the bootstrap classloader to be accessible inside instrumentations.
+ */
+public final class GlobalMetricsTags {
+  private static final Logger log = LoggerFactory.getLogger(GlobalMetricsTags.class);
+
+  private static final List<Tag> EMPTY = Collections.emptyList();
+  private static final AtomicReference<List<Tag>> INSTANCE = new AtomicReference<>(EMPTY);
+
+  public static void set(List<Tag> globalTags) {
+    List<Tag> globalTagsCopy = Collections.unmodifiableList(new ArrayList<>(globalTags));
+    if (!INSTANCE.compareAndSet(EMPTY, globalTagsCopy)) {
+      log.warn("GlobalMetricTags#set() was already called before");
+    }
+  }
+
+  public static List<Tag> get() {
+    return INSTANCE.get();
+  }
+
+  private GlobalMetricsTags() {}
+}

build.gradle

@@ -20,7 +20,8 @@ def versions = [
     'opentelemetryJavaagent'     : snapshot ? "1.1.0-SNAPSHOT"       : '1.0.0',
     'opentelemetryJavaagentAlpha': snapshot ? "1.1.0-alpha-SNAPSHOT" : '1.0.0-alpha',
     'mockito'                    : '3.8.0',
-    'jupiter'                    : '5.7.1'
+    'jupiter'                    : '5.7.1',
+    'micrometer'                 : '1.6.4'
 ]
 
 subprojects {

custom/build.gradle

@@ -10,15 +10,28 @@ def relocatePackages = ext.relocatePackages
 
 dependencies {
   compileOnly(project(":bootstrap"))
-  implementation("io.opentelemetry:opentelemetry-sdk:${versions["opentelemetry"]}")
-  implementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:${versions.opentelemetryAlpha}")
-  implementation("io.opentelemetry:opentelemetry-exporter-jaeger-thrift:${versions.opentelemetry}")
-  implementation("io.opentelemetry.javaagent:opentelemetry-javaagent-spi:${versions.opentelemetryJavaagentAlpha}")
-  implementation("io.jaegertracing:jaeger-client:1.5.0")
+  compileOnly("io.opentelemetry:opentelemetry-sdk:${versions.opentelemetry}")
+  compileOnly("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:${versions.opentelemetryAlpha}")
+  compileOnly("io.opentelemetry:opentelemetry-semconv:${versions.opentelemetryAlpha}")
+  compileOnly("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:${versions.opentelemetryJavaagentAlpha}")
+  compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-spi:${versions.opentelemetryJavaagentAlpha}")
   annotationProcessor("com.google.auto.service:auto-service:1.0-rc7")
   annotationProcessor("com.google.auto:auto-common:0.8")
-  implementation("com.google.auto.service:auto-service:1.0-rc7")
-  implementation("com.google.auto:auto-common:0.8")
+  compileOnly("com.google.auto.service:auto-service:1.0-rc7")
+  compileOnly("com.google.auto:auto-common:0.8")
+
+  implementation("io.opentelemetry:opentelemetry-exporter-jaeger-thrift:${versions.opentelemetry}")
+  implementation("io.jaegertracing:jaeger-client:1.5.0")
+
+  compileOnly("io.micrometer:micrometer-core:${versions.micrometer}")
+  implementation("io.micrometer:micrometer-registry-signalfx:${versions.micrometer}") {
+    // bootstrap already has micrometer-core
+    exclude(group: 'io.micrometer', module: 'micrometer-core')
+  }
+
+  testImplementation("io.opentelemetry:opentelemetry-sdk:${versions.opentelemetry}")
+  testImplementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:${versions.opentelemetryJavaagentAlpha}")
+  testImplementation("io.micrometer:micrometer-core:${versions.micrometer}")
 }
 
 tasks {

custom/src/main/java/com/splunk/opentelemetry/SplunkConfiguration.java

@@ -31,6 +31,8 @@ public Map<String, String> getProperties() {
 
     // by default no metrics are exported
     config.put("otel.metrics.exporter", "none");
+    // disable otel runtime-metrics instrumentation; we use micrometer metrics instead
+    config.put("otel.instrumentation.runtime-metrics.enabled", "false");
 
     config.put("otel.traces.exporter", "jaeger-thrift-splunk");
     // http://localhost:9080/v1/trace is the default endpoint for SmartAgent

custom/src/main/java/com/splunk/opentelemetry/micrometer/GlobalTagsBuilder.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Splunk Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.splunk.opentelemetry.micrometer;
+
+import io.micrometer.core.instrument.Tag;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.sdk.resources.Resource;
+import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
+import java.util.ArrayList;
+import java.util.List;
+
+class GlobalTagsBuilder {
+  private final Resource resource;
+
+  GlobalTagsBuilder(Resource resource) {
+    this.resource = resource;
+  }
+
+  List<Tag> build() {
+    List<Tag> globalTags = new ArrayList<>(4);
+    addTag(globalTags, "deployment.environment", AttributeKey.stringKey("environment"));
+    addTag(globalTags, "service", ResourceAttributes.SERVICE_NAME);
+    addTag(globalTags, "runtime", ResourceAttributes.PROCESS_RUNTIME_NAME);
+    addTag(globalTags, "process.pid", ResourceAttributes.PROCESS_PID);
+    return globalTags;
+  }
+
+  private void addTag(List<Tag> tags, String tagName, AttributeKey<?> resourceAttributeKey) {
+    Object value = resource.getAttributes().get(resourceAttributeKey);
+    if (value != null) {
+      tags.add(Tag.of(tagName, value.toString()));
+    }
+  }
+}

custom/src/main/java/com/splunk/opentelemetry/micrometer/MicrometerBootstrapPackagesProvider.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Splunk Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.splunk.opentelemetry.micrometer;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.spi.BootstrapPackagesProvider;
+import java.util.Arrays;
+import java.util.List;
+
+@AutoService(BootstrapPackagesProvider.class)
+public class MicrometerBootstrapPackagesProvider implements BootstrapPackagesProvider {
+  @Override
+  public List<String> getPackagePrefixes() {
+    return Arrays.asList(
+        // IMPORTANT: must be io.micrometer.core, because io.micrometer.signalfx needs to be in the
+        // agent classloader
+        "io.micrometer.core", "org.HdrHistogram", "org.LatencyUtils");
+  }
+}

custom/src/main/java/com/splunk/opentelemetry/micrometer/MicrometerInstaller.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Splunk Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.splunk.opentelemetry.micrometer;
+
+import com.google.auto.service.AutoService;
+import com.splunk.opentelemetry.javaagent.bootstrap.GlobalMetricsTags;
+import io.micrometer.core.instrument.Clock;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.config.NamingConvention;
+import io.micrometer.signalfx.SignalFxMeterRegistry;
+import io.opentelemetry.instrumentation.api.config.Config;
+import io.opentelemetry.javaagent.spi.ComponentInstaller;
+import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkAutoConfiguration;
+import io.opentelemetry.sdk.resources.Resource;
+
+@AutoService(ComponentInstaller.class)
+public class MicrometerInstaller implements ComponentInstaller {
+  @Override
+  public void beforeByteBuddyAgent() {
+    Resource resource = OpenTelemetrySdkAutoConfiguration.getResource();
+    GlobalMetricsTags.set(new GlobalTagsBuilder(resource).build());
+    Metrics.addRegistry(createSplunkMeterRegistry(resource));
+  }
+
+  private static SignalFxMeterRegistry createSplunkMeterRegistry(Resource resource) {
+    SignalFxMeterRegistry signalFxRegistry =
+        new SignalFxMeterRegistry(new SplunkMetricsConfig(Config.get(), resource), Clock.SYSTEM);
+    NamingConvention signalFxNamingConvention = signalFxRegistry.config().namingConvention();
+    signalFxRegistry.config().namingConvention(new OtelNamingConvention(signalFxNamingConvention));
+    return signalFxRegistry;
+  }
+}

custom/src/main/java/com/splunk/opentelemetry/micrometer/OtelNamingConvention.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Splunk Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.splunk.opentelemetry.micrometer;
+
+import io.micrometer.core.instrument.Meter;
+import io.micrometer.core.instrument.config.NamingConvention;
+
+/**
+ * This class adds {@code runtime.} prefix to all JVM metrics produced by micrometer (they all begin
+ * with {@code jvm.} by default). There are two reasons to do that:
+ *
+ * <ol>
+ *   <li>To match what OTel metrics spec says about runtime metrics: and pretty much the only thing
+ *       that's specified there is that runtime metrics should start with {@code
+ *       runtime.{environment}.}
+ *   <li>To avoid conflicts with the {@code opentelemetry-java-contrib/jmx-metrics} tool that starts
+ *       all metrics with {@code jvm.}
+ * </ol>
+ */
+class OtelNamingConvention implements NamingConvention {
+  private final NamingConvention delegate;
+
+  OtelNamingConvention(NamingConvention delegate) {
+    this.delegate = delegate;
+  }
+
+  @Override
+  public String name(String name, Meter.Type type, String baseUnit) {
+    if (name.startsWith("jvm.")) {
+      name = "runtime." + name;
+    }
+    return delegate.name(name, type, baseUnit);
+  }
+
+  @Override
+  public String tagKey(String key) {
+    return delegate.tagKey(key);
+  }
+
+  @Override
+  public String tagValue(String value) {
+    return delegate.tagValue(value);
+  }
+}

custom/src/main/java/com/splunk/opentelemetry/micrometer/SplunkMetricsConfig.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Splunk Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.splunk.opentelemetry.micrometer;
+
+import io.micrometer.signalfx.SignalFxConfig;
+import io.opentelemetry.instrumentation.api.config.Config;
+import io.opentelemetry.sdk.resources.Resource;
+import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
+import java.time.Duration;
+
+class SplunkMetricsConfig implements SignalFxConfig {
+  static final String METRICS_ENABLED_PROPERTY = "splunk.metrics.enabled";
+  static final String ACCESS_TOKEN_PROPERTY = "splunk.access.token";
+  static final String METRICS_ENDPOINT_PROPERTY = "splunk.metrics.endpoint";
+  static final String METRICS_EXPORT_INTERVAL_PROPERTY = "splunk.metrics.export.interval";
+
+  // right now the default value points to SmartAgent endpoint
+  static final String DEFAULT_METRICS_ENDPOINT = "http://localhost:9080/v2/datapoint";
+  private static final String DEFAULT_METRICS_EXPORT_INTERVAL_MILLIS = "30000";
+
+  private final Config config;
+  // config values that are retrieved multiple times are cached
+  private final String accessToken;
+  private final String source;
+  private final Duration step;
+
+  SplunkMetricsConfig(Config config, Resource resource) {
+    this.config = config;
+
+    // non-empty token MUST be provided; we can just send anything because collector/SmartAgent will
+    // use the real one
+    accessToken = config.getProperty(ACCESS_TOKEN_PROPERTY, "no-token");
+    source = resource.getAttributes().get(ResourceAttributes.SERVICE_NAME);
+    step =
+        Duration.ofMillis(
+            Long.parseLong(
+                config.getProperty(
+                    METRICS_EXPORT_INTERVAL_PROPERTY, DEFAULT_METRICS_EXPORT_INTERVAL_MILLIS)));
+  }
+
+  @Override
+  public boolean enabled() {
+    return config.getBooleanProperty(METRICS_ENABLED_PROPERTY, true);
+  }
+
+  @Override
+  public String accessToken() {
+    return accessToken;
+  }
+
+  @Override
+  public String uri() {
+    return config.getProperty(METRICS_ENDPOINT_PROPERTY, DEFAULT_METRICS_ENDPOINT);
+  }
+
+  @Override
+  public String source() {
+    return source;
+  }
+
+  @Override
+  public Duration step() {
+    return step;
+  }
+
+  // hide other micrometer settings
+  @Override
+  public String prefix() {
+    return "splunk.internal.metrics";
+  }
+
+  @Override
+  public String get(String key) {
+    return config.getProperty(key);
+  }
+}