NH-2314: W3C trace context propagation

appoptics-opentelemetry-sdk/src/main/java/com/appoptics/api/ext/impl/TraceHandler.java

@@ -184,7 +184,7 @@ public void logException(Throwable error) {
      * {@inheritDoc}
      */
     public String getCurrentXTraceID() {
-        return Util.buildXTraceId(Span.current().getSpanContext());
+        return Util.W3CContextToHexString(Span.current().getSpanContext());
     }
 
     @Override

core-bootstrap/src/main/java/com/appoptics/opentelemetry/core/Constants.java

@@ -1,10 +1,13 @@
 package com.appoptics.opentelemetry.core;
 
 public interface Constants {
-    String AO_INTERNAL_ATTRIBUTE_PREFIX = "ao.internal.";
-    String AO_DETAILED_TRACING = AO_INTERNAL_ATTRIBUTE_PREFIX + "detailedTracing";
-    String AO_METRICS = AO_INTERNAL_ATTRIBUTE_PREFIX + "metrics";
-    String AO_SAMPLER = AO_INTERNAL_ATTRIBUTE_PREFIX + "sampler";
-    String AO_KEY_PREFIX = "ao.";
+    String SW_KEY_PREFIX = "sw.";
     String OT_KEY_PREFIX = "ot.";
+    String SW_INTERNAL_ATTRIBUTE_PREFIX = SW_KEY_PREFIX + "internal.";
+    String SW_DETAILED_TRACING = SW_INTERNAL_ATTRIBUTE_PREFIX + "detailedTracing";
+    String SW_METRICS = SW_INTERNAL_ATTRIBUTE_PREFIX + "metrics";
+    String SW_SAMPLER = SW_INTERNAL_ATTRIBUTE_PREFIX + "sampler";
+    String W3C_KEY_PREFIX = "w3c.";
+    String SW_UPSTREAM_TRACESTATE = SW_KEY_PREFIX + W3C_KEY_PREFIX + "tracestate";
+    String SW_PARENT_ID = SW_KEY_PREFIX + "parent_id";
 }

core-bootstrap/src/main/java/com/appoptics/opentelemetry/core/Util.java

@@ -1,9 +1,7 @@
 package com.appoptics.opentelemetry.core;
 
-import com.tracelytics.ext.google.common.base.Strings;
 import com.tracelytics.joboe.Metadata;
 import com.tracelytics.joboe.OboeException;
-import com.tracelytics.joboe.Constants;
 import io.opentelemetry.api.trace.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -17,29 +15,22 @@
 import static com.appoptics.opentelemetry.core.Constants.*;
 
 public class Util {
-    private static Logger logger = LoggerFactory.getLogger(Util.class.getName());
-    private static String APPOPTICS_TRACE_STATE_KEY = "appoptics";
-    private static byte EXIT_OP_ID_MASK = 0xf;
+    private static final Logger logger = LoggerFactory.getLogger(Util.class.getName());
+    private static final byte EXIT_OP_ID_MASK = 0xf;
 
-    /**
-     * Build an AO x-trace ID from OT span context. Take note that we will have to check the value of `appoptics` in
-     * `tracestate` first as AO x-trace ID has longer task ID than its OT counterpart traceid. More details in
-     * https://github.com/librato/joboe/issues/1079
+    /** Converts an OpenTelemetry span context to a hex string.
+     *
      * @param context
      * @return
      */
-    public static String buildXTraceId(SpanContext context) {
-        String aoId = context.getTraceState().get(APPOPTICS_TRACE_STATE_KEY);
-
-        String traceId = context.getTraceId();
-        if (aoId != null) {
-            try {
-                traceId = new Metadata(aoId).taskHexString();
-            } catch (OboeException e) {
-                logger.warn("Failed to convert appoptics trace state [" + aoId + "] to OT trace id", e);
-            }
-        }
-        return buildXTraceId(traceId, context.getSpanId(), context.isSampled());
+    public static String W3CContextToHexString(SpanContext context) {
+        return Metadata.CURRENT_VERSION_HEXSTRING
+                + Metadata.HEXSTRING_DELIMETER
+                + context.getTraceId()
+                + Metadata.HEXSTRING_DELIMETER
+                + context.getSpanId()
+                + Metadata.HEXSTRING_DELIMETER
+                + context.getTraceFlags().asHex();
     }
 
     /**
@@ -59,25 +50,6 @@ public static Metadata buildSpanExitMetadata(SpanContext context) {
         return exitContext;
     }
 
-    /**
-     * Builds an AO x-trace ID with OT trace id and span id
-     *
-     * @param traceId
-     * @param spanId
-     * @param isSampled
-     * @return
-     */
-    public static String buildXTraceId(String traceId, String spanId, boolean isSampled) {
-        final String HEADER = "2B";
-        String hexString = HEADER +
-                Strings.padEnd(traceId, Constants.MAX_TASK_ID_LEN * 2, '0') +
-                Strings.padEnd(spanId, Constants.MAX_OP_ID_LEN * 2, '0');
-        hexString += isSampled ? "01" : "00";
-
-
-        return hexString.toUpperCase();
-    }
-
     /**
      * Builds an AO metadata with OT span context
      *
@@ -86,15 +58,12 @@ public static String buildXTraceId(String traceId, String spanId, boolean isSamp
      */
     public static Metadata buildMetadata(SpanContext context) {
         try {
-            Metadata metadata = new Metadata(buildXTraceId(context));
-            metadata.setTraceId(toTraceId(context.getTraceIdBytes()));
-            return metadata;
+            return new Metadata(W3CContextToHexString(context));
         } catch (OboeException e) {
             return null;
         }
     }
 
-
     /**
      * Generate a deterministic AO trace id from the OT trace id bytes
      * @param traceIdBytes
@@ -116,34 +85,18 @@ public static Long toTraceId(byte[] traceIdBytes) {
      * @return
      */
     public static SpanContext toSpanContext(String xTrace, boolean isRemote) {
-        W3TraceContextHolder w3TraceContext = toW3TraceContext(xTrace);
-        return isRemote
-                ? SpanContext.createFromRemoteParent(w3TraceContext.traceId, w3TraceContext.spanId, w3TraceContext.traceFlags, TraceState.getDefault())
-                : SpanContext.create(w3TraceContext.traceId, w3TraceContext.spanId, w3TraceContext.traceFlags, TraceState.getDefault());
-    }
-
-    /**
-     * Builds a w3c formatted trace context with AO x-trace ID
-     * @param xTrace
-     * @return
-     */
-    public static W3TraceContextHolder toW3TraceContext(String xTrace) {
-        Metadata metadata;
+        Metadata metadata = null;
         try {
             metadata = new Metadata(xTrace);
         } catch (OboeException e) {
-            e.printStackTrace();
-            return null;
+            return SpanContext.getInvalid();
         }
 
-        String w3TraceId = TraceId.fromBytes(metadata.getTaskID());
-        String w3SpanId = SpanId.fromBytes(metadata.getOpID());
-        TraceFlags w3TraceFlags = metadata.isSampled() ? TraceFlags.getSampled() : TraceFlags.getDefault();
-
-        return new W3TraceContextHolder(w3TraceId, w3SpanId, w3TraceFlags);
+        return isRemote
+                ? SpanContext.createFromRemoteParent(metadata.taskHexString(), metadata.opHexString(), TraceFlags.fromByte(metadata.getFlags()), TraceState.getDefault())
+                : SpanContext.create(metadata.taskHexString(), metadata.opHexString(), TraceFlags.fromByte(metadata.getFlags()), TraceState.getDefault());
     }
 
-
     public static String parsePath(String url) {
         if (url != null) {
             try {
@@ -155,19 +108,6 @@ public static String parsePath(String url) {
         return null;
     }
 
-
-    public static class W3TraceContextHolder {
-        public final String traceId;
-        public final String spanId;
-        public final TraceFlags traceFlags;
-
-        W3TraceContextHolder(String traceId, String spanId, TraceFlags traceFlags) {
-            this.traceId = traceId;
-            this.spanId = spanId;
-            this.traceFlags = traceFlags;
-        }
-    }
-
     public static Map<String, Object> keyValuePairsToMap(Object... keyValuePairs) {
         Map<String, Object> map = new HashMap<String, Object>();
         if (keyValuePairs.length % 2 == 1) {
@@ -198,8 +138,8 @@ public static void setSpanAttributes(Span span, Map<String, ?> attributes) {
         for (Map.Entry<String, ?> entry : attributes.entrySet()) {
             Object value = entry.getValue();
             String key = entry.getKey();
-            if (!key.startsWith(AO_KEY_PREFIX)) {
-                key = AO_KEY_PREFIX + key;
+            if (!key.startsWith(SW_KEY_PREFIX)) {
+                key = SW_KEY_PREFIX + key;
             }
             if (value instanceof String) {
                 span.setAttribute(key, (String) value);
@@ -229,8 +169,8 @@ public static void setSpanAttributes(SpanBuilder spanBuilder, Map<String, ?> att
         for (Map.Entry<String, ?> entry : attributes.entrySet()) {
             Object value = entry.getValue();
             String key = entry.getKey();
-            if (!key.startsWith(AO_KEY_PREFIX)) {
-                key = AO_KEY_PREFIX + key;
+            if (!key.startsWith(SW_KEY_PREFIX)) {
+                key = SW_KEY_PREFIX + key;
             }
 
             if (value instanceof String) {

custom/src/main/java/com/appoptics/opentelemetry/extensions/AppOpticsContextPropagator.java

@@ -0,0 +1,142 @@
+package com.appoptics.opentelemetry.extensions;
+
+import com.tracelytics.instrumentation.HeaderConstants;
+import com.tracelytics.joboe.XTraceOptions;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.TraceState;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.propagation.TextMapGetter;
+import io.opentelemetry.context.propagation.TextMapPropagator;
+import io.opentelemetry.context.propagation.TextMapSetter;
+
+import javax.annotation.Nullable;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class AppOpticsContextPropagator implements TextMapPropagator {
+    private static final String TRACE_STATE_APPOPTICS_KEY = "sw";
+    static final String TRACE_PARENT = "traceparent";
+    static final String TRACE_STATE = "tracestate";
+    static final String X_TRACE_OPTIONS = "X-Trace-Options";
+    static final String X_TRACE_OPTIONS_SIGNATURE = "X-Trace-Options-Signature";
+    private static final List<String> FIELDS =
+            Collections.unmodifiableList(Arrays.asList(TRACE_PARENT, TRACE_STATE, HeaderConstants.W3C_TRACE_CONTEXT_HEADER));
+    private static final int TRACESTATE_MAX_SIZE = 512;
+    private static final int TRACESTATE_MAX_MEMBERS = 32;
+    private static final int OVERSIZE_ENTRY_LENGTH = 129;
+    private static final String TRACESTATE_KEY_VALUE_DELIMITER = "=";
+    private static final String TRACESTATE_ENTRY_DELIMITER = ",";
+
+    @Override
+    public Collection<String> fields() {
+        return FIELDS;
+    }
+
+    /**
+     * Injects the both the AppOptics x-trace ID and the updated w3c `tracestate` with our x-trace ID prepended
+     * into the carrier with values provided by current context
+     * @param context
+     * @param carrier
+     * @param setter
+     * @param <C>
+     */
+    @Override
+    public <C> void inject(Context context, @Nullable C carrier, TextMapSetter<C> setter) {
+        SpanContext spanContext = Span.fromContext(context).getSpanContext();
+        if (carrier == null || !spanContext.isValid()) {
+            return;
+        }
+        //update trace state too: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#tracestate
+        //https://www.w3.org/TR/trace-context/#mutating-the-tracestate-field
+        TraceState traceState = spanContext.getTraceState();
+        String swTraceStateValue = spanContext.getSpanId() + "-" + (spanContext.isSampled() ? "01" : "00");
+        setter.set(carrier, TRACE_STATE, updateTraceState(traceState, swTraceStateValue));
+
+        String traceOptions = context.get(TriggerTraceContextKey.XTRACE_OPTIONS);
+        String traceOptionsSignature = context.get(TriggerTraceContextKey.XTRACE_OPTIONS_SIGNATURE);
+        if (traceOptions != null) {
+            setter.set(carrier, X_TRACE_OPTIONS, traceOptions);
+        }
+        if (traceOptionsSignature != null) {
+            setter.set(carrier, X_TRACE_OPTIONS_SIGNATURE, traceOptionsSignature);
+        }
+    }
+
+    /**
+     * Update tracestate with the new SW tracestate value and do some truncation if needed.
+     * @param traceState
+     * @param swTraceStateValue
+     * @return
+     */
+    private String updateTraceState(TraceState traceState, String swTraceStateValue) {
+        StringBuilder traceStateBuilder = new StringBuilder(TRACESTATE_MAX_SIZE);
+        traceStateBuilder.append(TRACE_STATE_APPOPTICS_KEY).append(TRACESTATE_KEY_VALUE_DELIMITER).append(swTraceStateValue);
+        AtomicInteger count = new AtomicInteger(1);
+
+        // calculate current length of the tracestate
+        AtomicInteger traceStateLength = new AtomicInteger(0);
+        traceState.forEach(
+                (key, value) -> {
+                    if (!TRACE_STATE_APPOPTICS_KEY.equals(key)) {
+                        traceStateLength.addAndGet(key.length());
+                        traceStateLength.addAndGet(TRACESTATE_KEY_VALUE_DELIMITER.length());
+                        traceStateLength.addAndGet(value.length());
+                        traceStateLength.addAndGet(TRACESTATE_ENTRY_DELIMITER.length());
+                    }
+                }
+        );
+
+        AtomicBoolean truncateLargeEntry = new AtomicBoolean(traceStateLength.get() + traceStateBuilder.length() > TRACESTATE_MAX_SIZE);
+        traceState.forEach(
+                (key, value) -> {
+                    if (!TRACE_STATE_APPOPTICS_KEY.equals(key)
+                            && count.get() < TRACESTATE_MAX_MEMBERS
+                            && traceStateBuilder.length() + TRACESTATE_ENTRY_DELIMITER.length() + key.length() + TRACESTATE_KEY_VALUE_DELIMITER.length() + value.length() <= TRACESTATE_MAX_SIZE) {
+                        if (key.length() + TRACESTATE_KEY_VALUE_DELIMITER.length() + value.length() >= OVERSIZE_ENTRY_LENGTH
+                        && truncateLargeEntry.get()) {
+                            truncateLargeEntry.set(false); // only truncate one oversize entry as SW tracestate entry is smaller than OVERSIZE_ENTRY_LENGTH
+                        } else {
+                            traceStateBuilder.append(TRACESTATE_ENTRY_DELIMITER)
+                                    .append(key)
+                                    .append(TRACESTATE_KEY_VALUE_DELIMITER)
+                                    .append(value);
+                            count.incrementAndGet();
+                        }
+                    }
+                });
+        return traceStateBuilder.toString();
+    }
+    /**
+     * Extract context from the carrier, first scanning for appoptics x-trace header.
+     * If not found, try the w3c `tracestate`
+     * @param context
+     * @param carrier
+     * @param getter
+     * @param <C>
+     * @return
+     */
+    @Override
+    public <C> Context extract(Context context, @Nullable C carrier, TextMapGetter<C> getter) {
+        String traceOptions = getter.get(carrier, X_TRACE_OPTIONS);
+        String traceOptionsSignature = getter.get(carrier, X_TRACE_OPTIONS_SIGNATURE);
+        XTraceOptions xTraceOptions = XTraceOptions.getXTraceOptions(traceOptions, traceOptionsSignature);
+        if (xTraceOptions != null) {
+            context = context.with(TriggerTraceContextKey.KEY, xTraceOptions);
+            context = context.with(TriggerTraceContextKey.XTRACE_OPTIONS, traceOptions);
+            if (traceOptionsSignature != null) {
+                context = context.with(TriggerTraceContextKey.XTRACE_OPTIONS_SIGNATURE, traceOptionsSignature);
+            }
+        }
+
+        String traceState = getter.get(carrier, TRACE_STATE);
+        if (traceState != null) {
+            context = context.with(TraceStateKey.KEY, traceState);
+        }
+        return context;
+    }
+}

custom/src/main/java/com/appoptics/opentelemetry/extensions/AppOpticsPropagatorProvider.java → custom/src/main/java/com/appoptics/opentelemetry/extensions/AppOpticsContextPropagatorProvider.java

@@ -5,15 +5,15 @@
 import io.opentelemetry.sdk.autoconfigure.spi.ConfigurablePropagatorProvider;
 
 @AutoService(ConfigurablePropagatorProvider.class)
-public class AppOpticsPropagatorProvider implements ConfigurablePropagatorProvider {
+public class AppOpticsContextPropagatorProvider implements ConfigurablePropagatorProvider {
 
     @Override
     public TextMapPropagator getPropagator() {
-        return new AppOpticsPropagator();
+        return new AppOpticsContextPropagator();
     }
 
     @Override
     public String getName() {
         return "appoptics";
     }
-}
+}

custom/src/main/java/com/appoptics/opentelemetry/extensions/AppOpticsInboundMetricsSpanProcessor.java

@@ -29,7 +29,7 @@
  * Span processor to record inbound metrics
  */
 public class AppOpticsInboundMetricsSpanProcessor implements SpanProcessor {
-    private static final AttributeKey<Boolean> AO_METRICS_KEY = AttributeKey.booleanKey(Constants.AO_METRICS);
+    private static final AttributeKey<Boolean> AO_METRICS_KEY = AttributeKey.booleanKey(Constants.SW_METRICS);
     public static final OpenTelemetryInboundMeasurementReporter measurementReporter = new OpenTelemetryInboundMeasurementReporter();
     public static final OpenTelemetryInboundHistogramReporter histogramReporter = new OpenTelemetryInboundHistogramReporter();
 

custom/src/main/java/com/appoptics/opentelemetry/extensions/AppOpticsProfilingSpanProcessor.java

@@ -36,9 +36,9 @@ public void onStart(Context parentContext, ReadWriteSpan span) {
                 if (PROFILER_ENABLED) {
                     Metadata metadata = Util.buildMetadata(span.getSpanContext());
                     Profiler.addProfiledThread(Thread.currentThread(), metadata, metadata.getTraceId());
-                    span.setAttribute(AO_KEY_PREFIX + "ProfileSpans", 1);
+                    span.setAttribute(SW_KEY_PREFIX + "ProfileSpans", 1);
                 } else {
-                    span.setAttribute(AO_KEY_PREFIX + "ProfileSpans", -1); //profiler disabled
+                    span.setAttribute(SW_KEY_PREFIX + "ProfileSpans", -1); //profiler disabled
                 }
             }
         }