SOLR-16938 Auto configure tracer without a <tracerConfig> tag in solr… (

#1922)

solr/CHANGES.txt

@@ -128,6 +128,8 @@ Improvements
 * SOLR-16461: `/solr/coreName/replication?command=backup` now has a v2 equivalent, available at
   `/api/cores/coreName/replication/backups` (Sanjay Dutt, Jason Gerlowski, Alex Deparvu)
 
+* SOLR-16938: Auto configure tracer without a <tracerConfig> tag in solr.xml (Alex Deparvu)
+
 Optimizations
 ---------------------
 

solr/core/src/java/org/apache/solr/core/TracerConfigurator.java

@@ -18,29 +18,98 @@
 package org.apache.solr.core;
 
 import io.opentelemetry.api.trace.Tracer;
+import java.lang.invoke.MethodHandles;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
 import org.apache.solr.util.tracing.SimplePropagator;
 import org.apache.solr.util.tracing.TraceUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** Produces a {@link Tracer} from configuration. */
 public abstract class TracerConfigurator implements NamedListInitializedPlugin {
 
   public static final boolean TRACE_ID_GEN_ENABLED =
       Boolean.parseBoolean(System.getProperty("solr.alwaysOnTraceId", "true"));
 
+  private static final String DEFAULT_CLASS_NAME =
+      System.getProperty(
+          "solr.otelDefaultConfigurator", "org.apache.solr.opentelemetry.OtelTracerConfigurator");
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
   public static Tracer loadTracer(SolrResourceLoader loader, PluginInfo info) {
     if (info != null && info.isEnabled()) {
       TracerConfigurator configurator =
           loader.newInstance(info.className, TracerConfigurator.class);
       configurator.init(info.initArgs);
       return configurator.getTracer();
-
-    } else if (TRACE_ID_GEN_ENABLED) {
+    }
+    if (shouldAutoConfigOTEL()) {
+      return autoConfigOTEL(loader);
+    }
+    if (TRACE_ID_GEN_ENABLED) {
       return SimplePropagator.load();
-    } else {
-      return TraceUtils.noop();
     }
+    return TraceUtils.getGlobalTracer();
   }
 
   protected abstract Tracer getTracer();
+
+  private static Tracer autoConfigOTEL(SolrResourceLoader loader) {
+    try {
+      TracerConfigurator configurator =
+          loader.newInstance(DEFAULT_CLASS_NAME, TracerConfigurator.class);
+      configurator.init(new NamedList<>());
+      return configurator.getTracer();
+    } catch (SolrException e) {
+      log.error(
+          "Unable to auto-config OpenTelemetry with class {}. Make sure you have enabled the 'opentelemetry' module",
+          DEFAULT_CLASS_NAME,
+          e);
+    }
+    return TraceUtils.getGlobalTracer();
+  }
+
+  /**
+   * Best effort way to determine if we should attempt to init OTEL from system properties.
+   *
+   * @return true if OTEL should be init
+   */
+  static boolean shouldAutoConfigOTEL() {
+    var env = System.getenv();
+    boolean isSdkDisabled = Boolean.parseBoolean(getConfig("OTEL_SDK_DISABLED", env));
+    if (isSdkDisabled) {
+      return false;
+    }
+    return getConfig("OTEL_SERVICE_NAME", env) != null;
+  }
+
+  /**
+   * Returns system property if found, else returns environment variable, or null if none found.
+   *
+   * @param envName the environment variable to look for
+   * @param env current env
+   * @return the resolved value
+   */
+  protected static String getConfig(String envName, Map<String, String> env) {
+    String sysName = envNameToSyspropName(envName);
+    String sysValue = System.getProperty(sysName);
+    String envValue = env.get(envName);
+    return sysValue != null ? sysValue : envValue;
+  }
+
+  /**
+   * In OTEL Java SDK there is a convention that the java property name for OTEL_FOO_BAR is
+   * otel.foo.bar
+   *
+   * @param envName the environmnet name to convert
+   * @return the corresponding sysprop name
+   */
+  protected static String envNameToSyspropName(String envName) {
+    return envName.toLowerCase(Locale.ROOT).replace("_", ".");
+  }
 }

solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java

@@ -64,7 +64,7 @@ public static synchronized Tracer load() {
       GlobalOpenTelemetry.set(otel);
       loaded = true;
     }
-    return GlobalOpenTelemetry.getTracer("solr");
+    return TraceUtils.getGlobalTracer();
   }
 
   public static TextMapPropagator getInstance() {

solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java

@@ -22,7 +22,6 @@
 import io.opentelemetry.api.trace.SpanBuilder;
 import io.opentelemetry.api.trace.SpanKind;
 import io.opentelemetry.api.trace.Tracer;
-import io.opentelemetry.api.trace.TracerProvider;
 import io.opentelemetry.context.Context;
 import io.opentelemetry.context.propagation.TextMapPropagator;
 import java.util.List;
@@ -36,7 +35,6 @@ public class TraceUtils {
 
   private static final String REQ_ATTR_TRACING_SPAN = Span.class.getName();
   private static final String REQ_ATTR_TRACING_TRACER = Tracer.class.getName();
-  private static final Tracer NOOP_TRACER = TracerProvider.noop().get(null);
 
   public static final String DEFAULT_SPAN_NAME = "http.request";
   public static final String WRITE_QUERY_RESPONSE_SPAN_NAME = "writeQueryResponse";
@@ -66,8 +64,8 @@ public class TraceUtils {
 
   public static final String TAG_DB_TYPE_SOLR = "solr";
 
-  public static Tracer noop() {
-    return NOOP_TRACER;
+  public static Tracer getGlobalTracer() {
+    return GlobalOpenTelemetry.getTracer("solr");
   }
 
   public static TextMapPropagator getTextMapPropagator() {

solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import io.opentelemetry.api.trace.TracerProvider;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.util.tracing.TraceUtils;
+import org.junit.Test;
+
+public class TestTracerConfigurator extends SolrTestCaseJ4 {
+
+  @Test
+  public void configuratorClassDoesNotExistTest() {
+    System.setProperty("otel.service.name", "something");
+    System.setProperty("solr.otelDefaultConfigurator", "configuratorClassDoesNotExistTest");
+    try {
+      assertTrue(TracerConfigurator.shouldAutoConfigOTEL());
+      SolrResourceLoader loader = new SolrResourceLoader(TEST_PATH().resolve("collection1"));
+      TracerConfigurator.loadTracer(loader, null);
+      assertEquals(
+          "Expecting noop otel after failure to auto-init",
+          TracerProvider.noop().get(null),
+          TraceUtils.getGlobalTracer());
+    } finally {
+      System.clearProperty("solr.otelDefaultConfigurator");
+      System.clearProperty("otel.service.name");
+    }
+  }
+
+  @Test
+  public void otelDisabledByProperty() {
+    System.setProperty("OTEL_SDK_DISABLED", "true");
+    try {
+      assertFalse(TracerConfigurator.shouldAutoConfigOTEL());
+    } finally {
+      System.clearProperty("OTEL_SDK_DISABLED");
+    }
+  }
+}

solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java

@@ -16,7 +16,6 @@
  */
 package org.apache.solr.opentelemetry;
 
-import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.trace.Tracer;
 import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
 import java.lang.invoke.MethodHandles;
@@ -28,6 +27,7 @@
 import java.util.stream.Collectors;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.TracerConfigurator;
+import org.apache.solr.util.tracing.TraceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -49,17 +49,17 @@ public OtelTracerConfigurator() {
 
   @Override
   public Tracer getTracer() {
-    // TODO remove reliance on global
-    return GlobalOpenTelemetry.getTracer("solr");
+    return TraceUtils.getGlobalTracer();
   }
 
   @Override
   public void init(NamedList<?> args) {
-    prepareConfiguration();
+    prepareConfiguration(args);
     AutoConfiguredOpenTelemetrySdk.initialize();
   }
 
-  void prepareConfiguration() {
+  void prepareConfiguration(NamedList<?> args) {
+    injectPluginSettingsIfNotConfigured(args);
     setDefaultIfNotConfigured("OTEL_SERVICE_NAME", "solr");
     setDefaultIfNotConfigured("OTEL_TRACES_EXPORTER", "otlp");
     setDefaultIfNotConfigured("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc");
@@ -82,6 +82,20 @@ void prepareConfiguration() {
     System.setProperty("otel.logs.exporter", "none");
   }
 
+  /**
+   * Will inject plugin configuration values into system properties if not already setup (existing
+   * system properties take precedence)
+   */
+  private void injectPluginSettingsIfNotConfigured(NamedList<?> args) {
+    args.forEach(
+        (k, v) -> {
+          var asSysName = envNameToSyspropName(k);
+          if (asSysName.startsWith("otel.")) {
+            setDefaultIfNotConfigured(asSysName, v.toString());
+          }
+        });
+  }
+
   /**
    * Add explicit tags statically to all traces, independent of request. Attributes with same name
    * supplied in ENV or SysProp will take precedence over attributes added in code.
@@ -138,24 +152,11 @@ Map<String, String> getCurrentOtelConfig() {
   /**
    * Returns system property if found, else returns environment variable, or null if none found.
    *
-   * @param envName the environment to look for
+   * @param envName the environment variable to look for
    * @return the resolved value
    */
   String getEnvOrSysprop(String envName) {
-    String envValue = currentEnv.get(envName);
-    String propValue = System.getProperty(envNameToSyspropName(envName));
-    return propValue != null ? propValue : envValue;
-  }
-
-  /**
-   * In OTEL Java SDK there is a convention that the java property name for OTEL_FOO_BAR is
-   * otel.foo.bar
-   *
-   * @param envName the environmnet name to convert
-   * @return the corresponding sysprop name
-   */
-  static String envNameToSyspropName(String envName) {
-    return envName.toLowerCase(Locale.ROOT).replace("_", ".");
+    return getConfig(envName, currentEnv);
   }
 
   /**

solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java

@@ -62,7 +62,7 @@ public static synchronized void prepareForTest() {
 
     // force early init
     CustomTestOtelTracerConfigurator tracer = new CustomTestOtelTracerConfigurator();
-    tracer.prepareConfiguration();
+    tracer.prepareConfiguration(new NamedList<>());
 
     bootOtel();
   }

solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java

@@ -19,6 +19,7 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.util.NamedList;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -88,9 +89,19 @@ public void testSetDefaultIfNotConfigured() {
   @Test
   public void testResourceAttributes() throws Exception {
     System.setProperty("otel.resource.attributes", "foo=bar,ILLEGAL-LACKS-VALUE,");
-    instance.prepareConfiguration();
+    instance.prepareConfiguration(new NamedList<>());
     assertEquals(
         List.of("host.name=my.solr.host", "foo=bar"),
         List.of(System.getProperty("otel.resource.attributes").split(",")));
   }
+
+  @Test
+  public void testPluginConfig() throws Exception {
+    NamedList<String> conf = new NamedList<>();
+    conf.add("OTEL_K1", "conf-k1"); // will be replaced by sys prop
+    conf.add("otel.k7", "conf-k7"); // will be kept
+    instance.prepareConfiguration(conf);
+    assertEquals("prop-k1", instance.getCurrentOtelConfig().get("OTEL_K1"));
+    assertEquals("conf-k7", instance.getCurrentOtelConfig().get("OTEL_K7"));
+  }
 }

solr/server/resources/log4j2.xml

@@ -23,7 +23,7 @@
     <Console name="STDOUT" target="SYSTEM_OUT">
       <PatternLayout>
         <Pattern>
-          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{t:%X{trace_id}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
         </Pattern>
       </PatternLayout>
     </Console>
@@ -34,7 +34,7 @@
         filePattern="${sys:solr.log.dir}/solr.log.%i" >
       <PatternLayout>
         <Pattern>
-          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{t:%X{trace_id}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
         </Pattern>
       </PatternLayout>
       <Policies>
@@ -50,7 +50,7 @@
         filePattern="${sys:solr.log.dir}/solr_slow_requests.log.%i" >
       <PatternLayout>
         <Pattern>
-          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{t:%X{trace_id}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
         </Pattern>
       </PatternLayout>
       <Policies>

solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc

@@ -62,6 +62,8 @@ This module brings support for the industry standard https://opentelemetry.io[Op
 <tracerConfig name="tracerConfig" class="org.apache.solr.opentelemetry.OtelTracerConfigurator"/>
 ----
 
+As an alternative to changing the `solr.xml` file, the `OTEL` tracer will be enabled if the system property `otel.service.name` or environment variable `OTEL_SERVICE_NAME` is present. The `opentelemetry` module still needs to be enabled for the tracer to work.
+
 Enable the module with either system property `-Dsolr.modules=opentelemetry` or environment variable `SOLR_MODULES=opentelemetry`.
 
 === Configuration

solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java

@@ -1288,6 +1288,13 @@ public Builder withDefaultClusterProperty(String key, String value) {
       return this;
     }
 
+    /**
+     * Disables the default/built-in simple trace ID generation/propagation.
+     *
+     * <p>Tracers are registered as global singletons and if for example a test needs to use a
+     * MockTracer or a "real" Tracer, it needs to call this method so that the test setup doesn't
+     * accidentally reset the Tracer it wants to use.
+     */
     public Builder withTraceIdGenerationDisabled() {
       this.disableTraceIdGeneration = true;
       return this;