Extract HTTP request/response headers as span attributes (#4237)

* Extract HTTP request/response headers as span attributes

* fix muzzle

* code review comments

* fix compilation failure after merge conflict

* avoid using streams API when transforming the headers list

* fix liberty extractor

* fix spring webmvc extractor

benchmark/src/jmh/java/io/opentelemetry/benchmark/InstrumenterBenchmark.java

@@ -8,11 +8,14 @@
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.context.Context;
 import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.api.instrumenter.http.CapturedHttpHeaders;
 import io.opentelemetry.instrumentation.api.instrumenter.http.HttpClientAttributesExtractor;
 import io.opentelemetry.instrumentation.api.instrumenter.http.HttpSpanNameExtractor;
 import io.opentelemetry.instrumentation.api.instrumenter.net.InetSocketAddressNetAttributesExtractor;
 import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
 import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -58,6 +61,10 @@ static class ConstantHttpAttributesExtractor extends HttpClientAttributesExtract
     static final HttpClientAttributesExtractor<Void, Void> INSTANCE =
         new ConstantHttpAttributesExtractor();
 
+    public ConstantHttpAttributesExtractor() {
+      super(CapturedHttpHeaders.empty());
+    }
+
     @Override
     protected @Nullable String method(Void unused) {
       return "GET";
@@ -73,6 +80,11 @@ static class ConstantHttpAttributesExtractor extends HttpClientAttributesExtract
       return "OpenTelemetryBot";
     }
 
+    @Override
+    protected List<String> requestHeader(Void unused, String name) {
+      return Collections.emptyList();
+    }
+
     @Override
     protected @Nullable Long requestContentLength(Void unused, @Nullable Void unused2) {
       return 100L;
@@ -102,6 +114,11 @@ static class ConstantHttpAttributesExtractor extends HttpClientAttributesExtract
     protected @Nullable Long responseContentLengthUncompressed(Void unused, Void unused2) {
       return null;
     }
+
+    @Override
+    protected List<String> responseHeader(Void unused, Void unused2, String name) {
+      return Collections.emptyList();
+    }
   }
 
   static class ConstantNetAttributesExtractor

instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/ServerInstrumenter.java

@@ -34,6 +34,8 @@ public Context start(Context parentContext, REQUEST request) {
     return super.start(extracted, request);
   }
 
+  // TODO: move that to HttpServerAttributesExtractor, now that we have a method for extracting
+  // header values there
   private static <REQUEST, RESPONSE> InstrumenterBuilder<REQUEST, RESPONSE> addClientIpExtractor(
       InstrumenterBuilder<REQUEST, RESPONSE> builder, TextMapGetter<REQUEST> getter) {
     HttpServerAttributesExtractor<REQUEST, RESPONSE> httpAttributesExtractor = null;

instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/http/CapturedHttpHeaders.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.instrumenter.http;
+
+import static java.util.Collections.emptyList;
+
+import com.google.auto.value.AutoValue;
+import java.util.List;
+import java.util.Locale;
+import java.util.stream.Collectors;
+
+/**
+ * Represents the configuration that specifies which HTTP request/response headers should be
+ * captured as span attributes.
+ */
+@AutoValue
+public abstract class CapturedHttpHeaders {
+
+  private static final CapturedHttpHeaders EMPTY = create(emptyList(), emptyList());
+
+  /** Don't capture any HTTP headers as span attributes. */
+  public static CapturedHttpHeaders empty() {
+    return EMPTY;
+  }
+
+  /**
+   * Captures the configured HTTP request and response headers as span attributes as described in <a
+   * href="https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-request-and-response-headers">HTTP
+   * semantic conventions</a>.
+   *
+   * <p>The HTTP request header values will be captured under the {@code http.request.header.<name>}
+   * attribute key. The HTTP response header values will be captured under the {@code
+   * http.response.header.<name>} attribute key. The {@code <name>} part in the attribute key is the
+   * normalized header name: lowercase, with dashes replaced by underscores.
+   *
+   * @param capturedRequestHeaders A list of HTTP request header names that are to be captured as
+   *     span attributes.
+   * @param capturedResponseHeaders A list of HTTP response header names that are to be captured as
+   *     span attributes.
+   */
+  public static CapturedHttpHeaders create(
+      List<String> capturedRequestHeaders, List<String> capturedResponseHeaders) {
+    return new AutoValue_CapturedHttpHeaders(
+        lowercase(capturedRequestHeaders), lowercase(capturedResponseHeaders));
+  }
+
+  private static List<String> lowercase(List<String> names) {
+    return names.stream().map(s -> s.toLowerCase(Locale.ROOT)).collect(Collectors.toList());
+  }
+
+  /** Returns the list of HTTP request header names that are to be captured as span attributes. */
+  public abstract List<String> requestHeaders();
+
+  /** Returns the list of HTTP response header names that are to be captured as span attributes. */
+  public abstract List<String> responseHeaders();
+
+  CapturedHttpHeaders() {}
+}

instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/http/HttpClientAttributesExtractor.java

@@ -23,6 +23,16 @@
 public abstract class HttpClientAttributesExtractor<REQUEST, RESPONSE>
     extends HttpCommonAttributesExtractor<REQUEST, RESPONSE> {
 
+  /**
+   * Create the HTTP client attributes extractor.
+   *
+   * @param capturedHttpHeaders A configuration object specifying which HTTP request and response
+   *     headers should be captured as span attributes.
+   */
+  protected HttpClientAttributesExtractor(CapturedHttpHeaders capturedHttpHeaders) {
+    super(capturedHttpHeaders);
+  }
+
   @Override
   protected final void onStart(AttributesBuilder attributes, REQUEST request) {
     super.onStart(attributes, request);

instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/http/HttpCommonAttributesExtractor.java

@@ -5,11 +5,15 @@
 
 package io.opentelemetry.instrumentation.api.instrumenter.http;
 
+import static io.opentelemetry.instrumentation.api.instrumenter.http.HttpHeaderAttributes.requestAttributeKey;
+import static io.opentelemetry.instrumentation.api.instrumenter.http.HttpHeaderAttributes.responseAttributeKey;
+
 import io.opentelemetry.api.common.AttributesBuilder;
 import io.opentelemetry.context.Context;
 import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
 import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
 import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
+import java.util.List;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 /**
@@ -20,13 +24,23 @@
 public abstract class HttpCommonAttributesExtractor<REQUEST, RESPONSE>
     extends AttributesExtractor<REQUEST, RESPONSE> {
 
-  // directly extending this class should not be possible outside of this package
-  HttpCommonAttributesExtractor() {}
+  private final CapturedHttpHeaders capturedHttpHeaders;
+
+  HttpCommonAttributesExtractor(CapturedHttpHeaders capturedHttpHeaders) {
+    this.capturedHttpHeaders = capturedHttpHeaders;
+  }
 
   @Override
   protected void onStart(AttributesBuilder attributes, REQUEST request) {
     set(attributes, SemanticAttributes.HTTP_METHOD, method(request));
     set(attributes, SemanticAttributes.HTTP_USER_AGENT, userAgent(request));
+
+    for (String name : capturedHttpHeaders.requestHeaders()) {
+      List<String> values = requestHeader(request, name);
+      if (!values.isEmpty()) {
+        set(attributes, requestAttributeKey(name), values);
+      }
+    }
   }
 
   @Override
@@ -44,6 +58,7 @@ protected void onEnd(
         attributes,
         SemanticAttributes.HTTP_REQUEST_CONTENT_LENGTH_UNCOMPRESSED,
         requestContentLengthUncompressed(request, response));
+
     if (response != null) {
       Integer statusCode = statusCode(request, response);
       if (statusCode != null) {
@@ -57,6 +72,13 @@ protected void onEnd(
           attributes,
           SemanticAttributes.HTTP_RESPONSE_CONTENT_LENGTH_UNCOMPRESSED,
           responseContentLengthUncompressed(request, response));
+
+      for (String name : capturedHttpHeaders.responseHeaders()) {
+        List<String> values = responseHeader(request, response, name);
+        if (!values.isEmpty()) {
+          set(attributes, responseAttributeKey(name), values);
+        }
+      }
     }
   }
 
@@ -65,8 +87,21 @@ protected void onEnd(
   @Nullable
   protected abstract String method(REQUEST request);
 
+  // TODO: remove implementations?
   @Nullable
-  protected abstract String userAgent(REQUEST request);
+  protected String userAgent(REQUEST request) {
+    List<String> values = requestHeader(request, "user-agent");
+    return values.isEmpty() ? null : values.get(0);
+  }
+
+  /**
+   * Extracts all values of header named {@code name} from the request, or an empty list if there
+   * were none.
+   *
+   * <p>Implementations of this method <b>must not</b> return a null value; an empty list should be
+   * returned instead.
+   */
+  protected abstract List<String> requestHeader(REQUEST request, String name);
 
   // Attributes which are not always available when the request is ready.
 
@@ -115,4 +150,16 @@ protected abstract Long requestContentLengthUncompressed(
    */
   @Nullable
   protected abstract Long responseContentLengthUncompressed(REQUEST request, RESPONSE response);
+
+  /**
+   * Extracts all values of header named {@code name} from the response, or an empty list if there
+   * were none.
+   *
+   * <p>This is called from {@link Instrumenter#end(Context, Object, Object, Throwable)}, only when
+   * {@code response} is non-{@code null}.
+   *
+   * <p>Implementations of this method <b>must not</b> return a null value; an empty list should be
+   * returned instead.
+   */
+  protected abstract List<String> responseHeader(REQUEST request, RESPONSE response, String name);
 }

instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/http/HttpHeaderAttributes.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.instrumenter.http;
+
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.instrumentation.api.caching.Cache;
+import java.util.List;
+
+final class HttpHeaderAttributes {
+
+  private static final Cache<String, AttributeKey<List<String>>> requestKeysCache =
+      Cache.newBuilder().setMaximumSize(32).build();
+  private static final Cache<String, AttributeKey<List<String>>> responseKeysCache =
+      Cache.newBuilder().setMaximumSize(32).build();
+
+  static AttributeKey<List<String>> requestAttributeKey(String headerName) {
+    return requestKeysCache.computeIfAbsent(headerName, n -> createKey("request", n));
+  }
+
+  static AttributeKey<List<String>> responseAttributeKey(String headerName) {
+    return responseKeysCache.computeIfAbsent(headerName, n -> createKey("response", n));
+  }
+
+  private static AttributeKey<List<String>> createKey(String type, String headerName) {
+    // headerName is always lowercase, see CapturedHttpHeaders
+    String key = "http." + type + ".header." + headerName.replace('-', '_');
+    return AttributeKey.stringArrayKey(key);
+  }
+
+  private HttpHeaderAttributes() {}
+}

instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/http/HttpServerAttributesExtractor.java

@@ -9,6 +9,7 @@
 import io.opentelemetry.context.Context;
 import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
 import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
+import java.util.List;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 /**
@@ -23,6 +24,16 @@
 public abstract class HttpServerAttributesExtractor<REQUEST, RESPONSE>
     extends HttpCommonAttributesExtractor<REQUEST, RESPONSE> {
 
+  /**
+   * Create the HTTP server attributes extractor.
+   *
+   * @param capturedHttpHeaders A configuration object specifying which HTTP request and response
+   *     headers should be captured as span attributes.
+   */
+  protected HttpServerAttributesExtractor(CapturedHttpHeaders capturedHttpHeaders) {
+    super(capturedHttpHeaders);
+  }
+
   @Override
   protected final void onStart(AttributesBuilder attributes, REQUEST request) {
     super.onStart(attributes, request);
@@ -53,8 +64,12 @@ protected final void onEnd(
   @Nullable
   protected abstract String target(REQUEST request);
 
+  // TODO: remove implementations?
   @Nullable
-  protected abstract String host(REQUEST request);
+  protected String host(REQUEST request) {
+    List<String> values = requestHeader(request, "host");
+    return values.isEmpty() ? null : values.get(0);
+  }
 
   @Nullable
   protected abstract String route(REQUEST request);

instrumentation-api/src/test/java/io/opentelemetry/instrumentation/api/instrumenter/InstrumenterTest.java

@@ -273,15 +273,12 @@ void server_http() {
                 otelTesting.getOpenTelemetry(), "test", unused -> "span")
             .addAttributesExtractors(
                 mockHttpServerAttributes,
-                mockNetAttributes,
+                new ConstantNetPeerIpExtractor<>("2.2.2.2"),
                 new AttributesExtractor1(),
                 new AttributesExtractor2())
             .addSpanLinksExtractor(new LinksExtractor())
             .newServerInstrumenter(new MapGetter());
 
-    when(mockNetAttributes.peerIp(REQUEST, null)).thenReturn("2.2.2.2");
-    when(mockNetAttributes.peerIp(REQUEST, RESPONSE)).thenReturn("2.2.2.2");
-
     Context context = instrumenter.start(Context.root(), REQUEST);
     SpanContext spanContext = Span.fromContext(context).getSpanContext();
 
@@ -312,7 +309,7 @@ void server_http_xForwardedFor() {
                 otelTesting.getOpenTelemetry(), "test", unused -> "span")
             .addAttributesExtractors(
                 mockHttpServerAttributes,
-                mockNetAttributes,
+                new ConstantNetPeerIpExtractor<>("2.2.2.2"),
                 new AttributesExtractor1(),
                 new AttributesExtractor2())
             .addSpanLinksExtractor(new LinksExtractor())
@@ -322,9 +319,6 @@ void server_http_xForwardedFor() {
     request.remove("Forwarded");
     request.put("X-Forwarded-For", "1.1.1.1");
 
-    when(mockNetAttributes.peerIp(request, null)).thenReturn("2.2.2.2");
-    when(mockNetAttributes.peerIp(request, RESPONSE)).thenReturn("2.2.2.2");
-
     Context context = instrumenter.start(Context.root(), request);
     SpanContext spanContext = Span.fromContext(context).getSpanContext();
 
@@ -928,4 +922,34 @@ private static LinkData expectedSpanLink() {
         SpanContext.create(
             LINK_TRACE_ID, LINK_SPAN_ID, TraceFlags.getSampled(), TraceState.getDefault()));
   }
+
+  private static final class ConstantNetPeerIpExtractor<REQUEST, RESPONSE>
+      extends NetAttributesExtractor<REQUEST, RESPONSE> {
+
+    private final String peerIp;
+
+    private ConstantNetPeerIpExtractor(String peerIp) {
+      this.peerIp = peerIp;
+    }
+
+    @Override
+    public String transport(REQUEST request) {
+      return null;
+    }
+
+    @Override
+    public String peerName(REQUEST request, RESPONSE response) {
+      return null;
+    }
+
+    @Override
+    public Integer peerPort(REQUEST request, RESPONSE response) {
+      return null;
+    }
+
+    @Override
+    public String peerIp(REQUEST request, RESPONSE response) {
+      return peerIp;
+    }
+  }
 }