FTV1 support (#1514)

Adds FTV1 support. A new open telemetry exporter has been added that will convert regular traces to Apollo traces. A buffer of spans is collected on the server side which will retain spans until the root request span is completed. Once a request is completed the trace will be reconstructed and sent to Apollo. Span attributes that are only relevant to Apollo tracing are prefixed with `apollo_private.` and are filtered out of other APM data. @glasser Has given some guidance on how we should improve tracing, but this'll be left to followup tickets as this PR is large and has been ongoing for a significant period. - [ ] [Don't send ftv1 traces to free tier users](#1728). - [ ] [Only send ftv1 traces that are interesting](#1729). As an aside, this PR demonstrates that spans can be used for Apollo tracing, and that we could move to a native Otel based solution in future. Signed-off-by: Benjamin Coenen <[email protected]> Co-authored-by: bryn <[email protected]> Co-authored-by: Benjamin Coenen <[email protected]> Co-authored-by: o0Ignition0o <[email protected]> Co-authored-by: Coenen Benjamin <[email protected]> Co-authored-by: Jesse Rosenberger <[email protected]> Co-authored-by: Gary Pennington <[email protected]>
apollographql · Sep 8, 2022 · 514bb86 · 514bb86
1 parent 64e057c
commit 514bb86
Show file tree

Hide file tree

Showing 44 changed files with 2,524 additions and 1,252 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -26,7 +26,45 @@ By [@USERNAME](https://github.com/USERNAME) in https://github.com/apollographql/
 # [x.x.x] (unreleased) - 2022-mm-dd
 
 ## ❗ BREAKING ❗
+### Span client_name and client_version attributes renamed ([#1514](https://github.com/apollographql/router/issues/1514))
+OpenTelemetry attributes should be grouped by `.` rather than `_`, therefore the following attributes have changed:
+
+* `client_name` => `client.name`
+* `client_version` => `client.version`
+
+By [@BrynCooke](https://github.com/BrynCooke) in https://github.com/apollographql/router/pull/1514
+
 ## 🚀 Features
+
+### Add federated tracing support to Apollo studio usage reporting ([#1514](https://github.com/apollographql/router/issues/1514))
+
+Add support of [federated tracing](https://www.apollographql.com/docs/federation/metrics/) in Apollo Studio:
+
+```yaml
+telemetry:
+    apollo:
+        # The percentage of requests will include HTTP request and response headers in traces sent to Apollo Studio.
+        # This is expensive and should be left at a low value.
+        # This cannot be higher than tracing->trace_config->sampler
+        field_level_instrumentation_sampler: 0.01 # (default)
+
+        # Include HTTP request and response headers in traces sent to Apollo Studio
+        send_headers: # other possible values are all, only (with an array), except (with an array), none (by default)
+            except: # Send all headers except referer
+            - referer
+
+        # Send variable values in Apollo in traces sent to Apollo Studio
+        send_variable_values: # other possible values are all, only (with an array), except (with an array), none (by default)
+            except: # Send all variable values except for variable named first
+            - first
+    tracing:
+        trace_config:
+            sampler: 0.5 # The percentage of requests that will generate traces (a rate or `always_on` or `always_off`)
+```
+
+By [@BrynCooke](https://github.com/BrynCooke) & [@bnjjj](https://github.com/bnjjj) & [@o0Ignition0o](https://github.com/o0Ignition0o) in https://github.com/apollographql/router/pull/1514
+
+
 ## 🐛 Fixes
 ## 🛠 Maintenance
 

diff --git a/about.toml b/about.toml
@@ -8,7 +8,8 @@ accepted = [
     "LicenseRef-ELv2",
     "LicenseRef-ring",
     "MIT",
-    "MPL-2.0"
+    "MPL-2.0",
+    "Unicode-DFS-2016"
 ]
 
 # Ignore non plublished crates, such as xtask for example

diff --git a/apollo-router-scaffold/src/plugin.rs b/apollo-router-scaffold/src/plugin.rs
@@ -98,7 +98,6 @@ fn create_plugin(name: &str, template_path: &Option<PathBuf>) -> Result<()> {
         Value::Boolean(true),
     );
 
-    dbg!(&params);
     desc.scaffold_with_parameters(params)?;
 
     let mod_path = mod_path();

diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml
@@ -34,7 +34,8 @@ async-trait = "0.1.57"
 atty = "0.2.14"
 axum = { version = "0.5.15", features = ["headers", "json", "original-uri"] }
 backtrace = "0.3.66"
-buildstructor = "0.4.1"
+base64 = "0.13.0"
+buildstructor = "0.5.0"
 bytes = "1.2.1"
 clap = { version = "3.2.20", default-features = false, features = [
     "env",
@@ -67,6 +68,7 @@ jsonschema = { version = "0.16.0", default-features = false }
 lazy_static = "1.4.0"
 libc = "0.2.132"
 lru = "0.7.8"
+mediatype = "0.19.9"
 mockall = "0.11.2"
 miette = { version = "5.3.0", features = ["fancy"] }
 mime = "0.3.16"
@@ -112,7 +114,9 @@ opentelemetry-zipkin = { version = "0.15.0", default-features = false, features
 ] }
 opentelemetry-prometheus = "0.10.0"
 paste = "1.0.9"
+pin-project-lite = "0.2.9"
 prometheus = "0.13"
+rand = "0.8.5"
 rhai = { version = "1.9.1", features = ["sync", "serde", "internals"] }
 regex = "1.6.0"
 reqwest = { version = "0.11.11", default-features = false, features = [
@@ -165,9 +169,8 @@ tracing-subscriber = { version = "0.3.11", features = ["env-filter", "json"] }
 
 url = { version = "2.3.0", features = ["serde"] }
 urlencoding = "2.1.2"
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 yaml-rust = "0.4.5"
-pin-project-lite = "0.2.9"
-mediatype = "0.19.9"
 
 [target.'cfg(macos)'.dependencies]
 uname = "0.1.1"
@@ -176,7 +179,7 @@ uname = "0.1.1"
 uname = "0.1.1"
 
 [dev-dependencies]
-insta = { version = "1.19.1", features = [ "json" ] }
+insta = { version = "1.19.1", features = [ "json", "redactions" ] }
 jsonpath_lib = "0.3.0"
 maplit = "1.0.2"
 mockall = "0.11.2"
@@ -199,8 +202,6 @@ tracing-subscriber = { version = "0.3", default-features = false, features = [
     "fmt",
 ] }
 tracing-test = "0.2.2"
-uuid = { version = "1.1.2", features = ["serde", "v4"] }
-url = "2.3.0"
 walkdir = "2.3.2"
 [[test]]
 name = "integration_tests"

diff --git a/apollo-router/src/axum_http_server_factory.rs b/apollo-router/src/axum_http_server_factory.rs
@@ -170,7 +170,9 @@ where
         .layer(
             TraceLayer::new_for_http()
                 .make_span_with(PropagatingMakeSpan::new())
-                .on_response(|resp: &Response<_>, _duration: Duration, span: &Span| {
+                .on_response(|resp: &Response<_>, duration: Duration, span: &Span| {
+                    // Duration here is instant based
+                    span.record("apollo_private.duration_ns", &(duration.as_nanos() as i64));
                     if resp.status() >= StatusCode::BAD_REQUEST {
                         span.record(
                             "otel.status_code",
@@ -821,6 +823,8 @@ impl PropagatingMakeSpan {
 
 impl<B> MakeSpan<B> for PropagatingMakeSpan {
     fn make_span(&mut self, request: &http::Request<B>) -> Span {
+        // This method needs to be moved to the telemetry plugin once we have a hook for the http request.
+
         // Before we make the span we need to attach span info that may have come in from the request.
         let context = global::get_text_map_propagator(|propagator| {
             propagator.extract(&opentelemetry_http::HeaderExtractor(request.headers()))
@@ -838,7 +842,8 @@ impl<B> MakeSpan<B> for PropagatingMakeSpan {
                 uri = %request.uri(),
                 version = ?request.version(),
                 "otel.kind" = %SpanKind::Server,
-                "otel.status_code" = %opentelemetry::trace::StatusCode::Unset.as_str()
+                "otel.status_code" = %opentelemetry::trace::StatusCode::Unset.as_str(),
+                "apollo_private.duration_ns" = tracing::field::Empty
             )
         } else {
             // No remote span, we can go ahead and create the span without context.
@@ -849,7 +854,8 @@ impl<B> MakeSpan<B> for PropagatingMakeSpan {
                 uri = %request.uri(),
                 version = ?request.version(),
                 "otel.kind" = %SpanKind::Server,
-                "otel.status_code" = %opentelemetry::trace::StatusCode::Unset.as_str()
+                "otel.status_code" = %opentelemetry::trace::StatusCode::Unset.as_str(),
+                "apollo_private.duration_ns" = tracing::field::Empty
             )
         }
     }

diff --git a/...r/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/...r/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap
@@ -468,19 +468,131 @@ expression: "&schema"
         "apollo": {
           "type": "object",
           "properties": {
+            "buffer_size": {
+              "description": "The buffer size for sending traces to Apollo. Increase this if you are experiencing lost traces.",
+              "default": 10000,
+              "type": "integer",
+              "format": "uint",
+              "minimum": 0.0
+            },
             "client_name_header": {
+              "description": "The name of the header to extract from requests when populating 'client nane' for traces and metrics in Apollo Studio.",
               "default": "apollographql-client-name",
               "type": "string",
               "nullable": true
             },
             "client_version_header": {
+              "description": "The name of the header to extract from requests when populating 'client version' for traces and metrics in Apollo Studio.",
               "default": "apollographql-client-version",
               "type": "string",
               "nullable": true
             },
             "endpoint": {
+              "description": "The Apollo Studio endpoint for exporting traces and metrics.",
               "type": "string",
               "nullable": true
+            },
+            "field_level_instrumentation_sampler": {
+              "description": "Enable field level instrumentation for subgraphs via ftv1. ftv1 tracing can cause performance issues as it is transmitted in band with subgraph responses. 0.0 will result in no field level instrumentation. 1.0 will result in always instrumentation. Value MUST be less than global sampling rate",
+              "anyOf": [
+                {
+                  "description": "Sample a given fraction. Fractions >= 1 will always sample.",
+                  "type": "number",
+                  "format": "double"
+                },
+                {
+                  "type": "string",
+                  "enum": [
+                    "always_on",
+                    "always_off"
+                  ]
+                }
+              ],
+              "nullable": true
+            },
+            "send_headers": {
+              "description": "To configure which request header names and values are included in trace data that's sent to Apollo Studio.",
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "none",
+                    "all"
+                  ]
+                },
+                {
+                  "type": "object",
+                  "required": [
+                    "only"
+                  ],
+                  "properties": {
+                    "only": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                {
+                  "type": "object",
+                  "required": [
+                    "except"
+                  ],
+                  "properties": {
+                    "except": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    }
+                  },
+                  "additionalProperties": false
+                }
+              ]
+            },
+            "send_variable_values": {
+              "description": "To configure which GraphQL variable values are included in trace data that's sent to Apollo Studio",
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "none",
+                    "all"
+                  ]
+                },
+                {
+                  "type": "object",
+                  "required": [
+                    "only"
+                  ],
+                  "properties": {
+                    "only": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                {
+                  "type": "object",
+                  "required": [
+                    "except"
+                  ],
+                  "properties": {
+                    "except": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    }
+                  },
+                  "additionalProperties": false
+                }
+              ]
             }
           },
           "additionalProperties": false,
@@ -1813,7 +1925,7 @@ expression: "&schema"
                 "sampler": {
                   "anyOf": [
                     {
-                      "description": "Sample a given fraction of traces. Fractions >= 1 will always sample. If the parent span is sampled, then it's child spans will automatically be sampled. Fractions < 0 are treated as zero, but spans may still be sampled if their parent is.",
+                      "description": "Sample a given fraction. Fractions >= 1 will always sample.",
                       "type": "number",
                       "format": "double"
                     },

diff --git a/apollo-router/src/context.rs b/apollo-router/src/context.rs
@@ -4,6 +4,7 @@
 //! allows additional data to be passed back and forth along the request invocation pipeline.
 
 use std::sync::Arc;
+use std::time::Instant;
 
 use dashmap::mapref::multiple::RefMulti;
 use dashmap::mapref::multiple::RefMutMulti;
@@ -30,13 +31,17 @@ pub(crate) type Entries = Arc<DashMap<String, Value>>;
 pub struct Context {
     // Allows adding custom entries to the context.
     entries: Entries,
+
+    /// Creation time
+    pub(crate) created_at: Instant,
 }
 
 impl Context {
     /// Create a new context.
     pub fn new() -> Self {
         Context {
             entries: Default::default(),
+            created_at: Instant::now(),
         }
     }
 }

diff --git a/apollo-router/src/http_ext.rs b/apollo-router/src/http_ext.rs
@@ -13,6 +13,7 @@ use axum::body::boxed;
 use axum::response::IntoResponse;
 use bytes::Bytes;
 use http::header;
+use http::header::HeaderName;
 use http::HeaderValue;
 use multimap::MultiMap;
 
@@ -260,11 +261,15 @@ impl<T> Request<T> {
         method: http::Method,
         body: T,
     ) -> http::Result<Request<T>> {
-        let mut req = http::request::Builder::new()
-            .method(method)
-            .uri(uri)
-            .body(body)?;
-        *req.headers_mut() = header_map(headers)?;
+        let mut builder = http::request::Builder::new().method(method).uri(uri);
+        for (key, values) in headers {
+            let header_name: HeaderName = key.try_into()?;
+            for value in values {
+                let header_value: HeaderValue = value.try_into()?;
+                builder = builder.header(header_name.clone(), header_value);
+            }
+        }
+        let req = builder.body(body)?;
         Ok(Self { inner: req })
     }
 }