From d0e9391968e7b2452adde9fe0becdc6d32417bb3 Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Mon, 11 Sep 2023 14:46:18 +0200 Subject: [PATCH 01/51] Update tokio-tungstenite (#3643) Co-authored-by: Coenen Benjamin --- Cargo.lock | 47 +++---------------- apollo-router/Cargo.toml | 4 +- .../src/services/subgraph_service.rs | 2 +- 3 files changed, 9 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ea7eb152b..6d20ce4600 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,7 +380,7 @@ dependencies = [ "tokio", "tokio-rustls 0.24.1", "tokio-stream", - "tokio-tungstenite 0.18.0", + "tokio-tungstenite", "tokio-util", "toml 0.7.6", "tonic 0.8.3", @@ -972,7 +972,7 @@ dependencies = [ "sha1 0.10.5", "sync_wrapper", "tokio", - "tokio-tungstenite 0.20.0", + "tokio-tungstenite", "tower", "tower-layer", "tower-service", @@ -6374,22 +6374,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "tokio-tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" -dependencies = [ - "futures-util", - "log", - "rustls 0.20.8", - "rustls-native-certs", - "tokio", - "tokio-rustls 0.23.4", - "tungstenite 0.18.0", - "webpki", -] - [[package]] name = "tokio-tungstenite" version = "0.20.0" @@ -6398,8 +6382,11 @@ checksum = "2b2dbec703c26b00d74844519606ef15d09a7d6857860f84ad223dec002ddea2" dependencies = [ "futures-util", "log", + "rustls 0.21.6", + "rustls-native-certs", "tokio", - "tungstenite 0.20.0", + "tokio-rustls 0.24.1", + "tungstenite", ] [[package]] @@ -6778,27 +6765,6 @@ dependencies = [ "syn 2.0.29", ] -[[package]] -name = "tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" -dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "http", - "httparse", - "log", - "rand 0.8.5", - "rustls 0.20.8", - "sha1 0.10.5", - "thiserror", - "url", - "utf-8", - "webpki", -] - [[package]] name = "tungstenite" version = "0.20.0" @@ -6812,6 +6778,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", + "rustls 0.21.6", "sha1 0.10.5", "thiserror", "url", diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 6e9eab0e03..8af25ed3df 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -226,9 +226,7 @@ uuid = { version = "1.4.1", features = ["serde", "v4"] } yaml-rust = "0.4.5" wiremock = "0.5.19" wsl = "0.1.0" -tokio-tungstenite = { version = "0.18.0", features = [ - "rustls-tls-native-roots", -] } +tokio-tungstenite = { version = "0.20.0", features = ["rustls-tls-native-roots"] } tokio-rustls = "0.24.1" http-serde = "1.1.3" hmac = "0.12.1" diff --git a/apollo-router/src/services/subgraph_service.rs b/apollo-router/src/services/subgraph_service.rs index 8ba73b4068..2d0044d488 100644 --- a/apollo-router/src/services/subgraph_service.rs +++ b/apollo-router/src/services/subgraph_service.rs @@ -542,7 +542,7 @@ async fn call_websocket( let (ws_stream, mut resp) = match request.uri().scheme_str() { Some("wss") => { - connect_async_tls_with_config(request, None, None) + connect_async_tls_with_config(request, None, false, None) .instrument(subgraph_req_span) .await } From cd3e4291fd81c45da6e3575e84dbacfa3fc2b18e Mon Sep 17 00:00:00 2001 From: Jesse Rosenberger Date: Tue, 12 Sep 2023 11:27:06 +0300 Subject: [PATCH 02/51] Update Changelog for #3586 (#3804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This corrects the CHANGELOG entry for #3586 which inadvertently suggested using `` twice instead of `` and (separately) ``: ``` https://studio.apollographql.com/graph//variant//operations?query= ``` This doesn't replace the need to document this in https://github.com/apollographql/router/issues/3803. 😄 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f26fb183..e3050a84b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -146,7 +146,7 @@ By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/p This exposes a new key in the `Context`, `apollo_operation_id`, which identifies operation you can find in studio: ``` -https://studio.apollographql.com/graph//variant//operations?query= +https://studio.apollographql.com/graph//variant//operations?query= ``` The `apollo_operation_id` context key is exposed during: From 38be5e29966955d7c4e976a2322bf85d50bec664 Mon Sep 17 00:00:00 2001 From: Gary Pennington Date: Tue, 12 Sep 2023 12:01:07 +0100 Subject: [PATCH 03/51] Helm: If there are extraLabels add them to all resources (#3622) This extends the functionality of `extraLabels` so that, if they are defined, they will be templated for all resources created by the chart. Previously, they were only templated onto the Deployment resource. --------- Co-authored-by: Bjoern Weidlich --- .changesets/fix_garypen_helm_extra_labels.md | 7 +++++++ helm/chart/router/templates/configmap.yaml | 3 +++ helm/chart/router/templates/deployment.yaml | 4 ++-- helm/chart/router/templates/hpa.yaml | 3 +++ helm/chart/router/templates/ingress.yaml | 3 +++ helm/chart/router/templates/pdb.yaml | 3 +++ helm/chart/router/templates/secret.yaml | 3 +++ helm/chart/router/templates/service.yaml | 3 +++ helm/chart/router/templates/serviceaccount.yaml | 3 +++ helm/chart/router/templates/serviceentry.yaml | 3 +++ helm/chart/router/templates/servicemonitor.yaml | 5 +++++ helm/chart/router/templates/supergraph-cm.yaml | 3 +++ helm/chart/router/templates/tests/test-connection.yaml | 3 +++ helm/chart/router/templates/virtualservice.yaml | 3 +++ helm/chart/router/values.yaml | 2 +- 15 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 .changesets/fix_garypen_helm_extra_labels.md diff --git a/.changesets/fix_garypen_helm_extra_labels.md b/.changesets/fix_garypen_helm_extra_labels.md new file mode 100644 index 0000000000..4082915ff1 --- /dev/null +++ b/.changesets/fix_garypen_helm_extra_labels.md @@ -0,0 +1,7 @@ +### Helm: If there are `extraLabels` add them to all resources ([PR #3622](https://github.com/apollographql/router/pull/3622)) + +This extends the functionality of `extraLabels` so that, if they are defined, they will be templated for all resources created by the chart. + +Previously, they were only templated onto the `Deployment` resource. + +By [@garypen](https://github.com/garypen) and [@bjoernw](https://github.com/bjoernw) in https://github.com/apollographql/router/pull/3622 diff --git a/helm/chart/router/templates/configmap.yaml b/helm/chart/router/templates/configmap.yaml index a51f1e6be3..e7caffecaf 100644 --- a/helm/chart/router/templates/configmap.yaml +++ b/helm/chart/router/templates/configmap.yaml @@ -9,6 +9,9 @@ metadata: name: {{ $routerFullName }} labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} data: configuration.yaml: | {{- toYaml $configuration | nindent 4 }} diff --git a/helm/chart/router/templates/deployment.yaml b/helm/chart/router/templates/deployment.yaml index dcda27697b..2fb6fa1c7a 100644 --- a/helm/chart/router/templates/deployment.yaml +++ b/helm/chart/router/templates/deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "router.labels" . | nindent 4 }} {{- if .Values.extraLabels }} - {{- include "common.templatizeExtraLabels" . | nindent 4 }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} {{- end }} {{/* There may not be much configuration so check that there is something */}} {{- if (((((.Values.router).configuration).telemetry).metrics).prometheus).enabled }} @@ -34,7 +34,7 @@ spec: labels: {{- include "router.selectorLabels" . | nindent 8 }} {{- if .Values.extraLabels }} - {{- include "common.templatizeExtraLabels" . | nindent 8 }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 8 }} {{- end }} spec: {{- with .Values.imagePullSecrets }} diff --git a/helm/chart/router/templates/hpa.yaml b/helm/chart/router/templates/hpa.yaml index 8a7891da03..b9e1eeb5a6 100644 --- a/helm/chart/router/templates/hpa.yaml +++ b/helm/chart/router/templates/hpa.yaml @@ -9,6 +9,9 @@ metadata: name: {{ include "router.fullname" . }} labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} spec: scaleTargetRef: apiVersion: apps/v1 diff --git a/helm/chart/router/templates/ingress.yaml b/helm/chart/router/templates/ingress.yaml index 355862b2d3..5f4bd976e1 100644 --- a/helm/chart/router/templates/ingress.yaml +++ b/helm/chart/router/templates/ingress.yaml @@ -7,6 +7,9 @@ metadata: name: {{ $fullName }} labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} {{- with .Values.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/chart/router/templates/pdb.yaml b/helm/chart/router/templates/pdb.yaml index 518fa5aea5..359a13af4d 100644 --- a/helm/chart/router/templates/pdb.yaml +++ b/helm/chart/router/templates/pdb.yaml @@ -6,6 +6,9 @@ metadata: name: {{ include "router.fullname" . }} labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} spec: {{- if .Values.podDisruptionBudget.minAvailable }} minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} diff --git a/helm/chart/router/templates/secret.yaml b/helm/chart/router/templates/secret.yaml index 3fbecd863e..9f5a4b124d 100644 --- a/helm/chart/router/templates/secret.yaml +++ b/helm/chart/router/templates/secret.yaml @@ -5,6 +5,9 @@ metadata: name: {{ template "router.managedFederation.apiSecretName" . }} labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} data: managedFederationApiKey: {{ default "MISSING" .Values.managedFederation.apiKey | b64enc | quote }} {{- end }} diff --git a/helm/chart/router/templates/service.yaml b/helm/chart/router/templates/service.yaml index 83990d91ef..57f59a2108 100644 --- a/helm/chart/router/templates/service.yaml +++ b/helm/chart/router/templates/service.yaml @@ -4,6 +4,9 @@ metadata: name: {{ include "router.fullname" . }} labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} {{- with .Values.service.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/chart/router/templates/serviceaccount.yaml b/helm/chart/router/templates/serviceaccount.yaml index 203f012bb0..b15b8eaf96 100644 --- a/helm/chart/router/templates/serviceaccount.yaml +++ b/helm/chart/router/templates/serviceaccount.yaml @@ -5,6 +5,9 @@ metadata: name: {{ include "router.serviceAccountName" . }} labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/chart/router/templates/serviceentry.yaml b/helm/chart/router/templates/serviceentry.yaml index 774eddeb31..ddc43407b4 100644 --- a/helm/chart/router/templates/serviceentry.yaml +++ b/helm/chart/router/templates/serviceentry.yaml @@ -12,6 +12,9 @@ metadata: labels: app.fullName: {{ $fullName }} {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} {{- with .Values.serviceentry.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/chart/router/templates/servicemonitor.yaml b/helm/chart/router/templates/servicemonitor.yaml index a4654a1add..61f9c88368 100644 --- a/helm/chart/router/templates/servicemonitor.yaml +++ b/helm/chart/router/templates/servicemonitor.yaml @@ -3,6 +3,11 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: {{ .Release.Name }} + labels: + {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} spec: endpoints: - path: {{ include "router.prometheusMetricsPath" . }} diff --git a/helm/chart/router/templates/supergraph-cm.yaml b/helm/chart/router/templates/supergraph-cm.yaml index 40ff9d073a..3a5f2a362a 100644 --- a/helm/chart/router/templates/supergraph-cm.yaml +++ b/helm/chart/router/templates/supergraph-cm.yaml @@ -6,6 +6,9 @@ metadata: name: {{ $routerFullName }}-supergraph labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} data: supergraph-schema.graphql: |- {{ .Values.supergraphFile | indent 4 }} diff --git a/helm/chart/router/templates/tests/test-connection.yaml b/helm/chart/router/templates/tests/test-connection.yaml index 0bf0612719..67b939bea0 100644 --- a/helm/chart/router/templates/tests/test-connection.yaml +++ b/helm/chart/router/templates/tests/test-connection.yaml @@ -4,6 +4,9 @@ metadata: name: "{{ include "router.fullname" . }}-test-connection" labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} annotations: "helm.sh/hook": test spec: diff --git a/helm/chart/router/templates/virtualservice.yaml b/helm/chart/router/templates/virtualservice.yaml index d8a77afede..fe3ef21db2 100644 --- a/helm/chart/router/templates/virtualservice.yaml +++ b/helm/chart/router/templates/virtualservice.yaml @@ -11,6 +11,9 @@ metadata: namespace: {{ .Values.virtualservice.namespace }} labels: {{- include "router.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{- include "common.templatizeExtraLabels" . | trim | nindent 4 }} + {{- end }} {{- with .Values.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/chart/router/values.yaml b/helm/chart/router/values.yaml index f682a3f933..94120f7f00 100644 --- a/helm/chart/router/values.yaml +++ b/helm/chart/router/values.yaml @@ -94,7 +94,7 @@ extraContainers: [] # command: ["sh"] initContainers: [] -# -- A map of extra labels to apply to the router deploment and containers +# -- A map of extra labels to apply to the resources created by this chart # Example: # extraLabels: # label_one_name: "label_one_value" From 826b840e0d0e5dd742e20a8f3af5c1a192f3d06b Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Tue, 12 Sep 2023 15:24:21 +0200 Subject: [PATCH 04/51] test(subscription): add integration test for subscription (#3752) Add a regression test that ensures an unnamed subscription makes it through the request pipeline. Fixes #3740 --------- Signed-off-by: Benjamin Coenen <5719034+bnjjj@users.noreply.github.com> --- .changesets/maint_bnjjj_fix_3740.md | 5 +++++ apollo-router/tests/common.rs | 18 +++++++++++---- .../tests/fixtures/subscription.router.yaml | 2 +- apollo-router/tests/subscription_load_test.rs | 22 ++++++++++++++++++- 4 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 .changesets/maint_bnjjj_fix_3740.md diff --git a/.changesets/maint_bnjjj_fix_3740.md b/.changesets/maint_bnjjj_fix_3740.md new file mode 100644 index 0000000000..d4bc5fee7d --- /dev/null +++ b/.changesets/maint_bnjjj_fix_3740.md @@ -0,0 +1,5 @@ +### Add integration test for subscription ([Issue #3740](https://github.com/apollographql/router/issues/3740)) + +Add a regression test that ensures an unnamed subscription makes it through the request pipeline. + +By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3752 \ No newline at end of file diff --git a/apollo-router/tests/common.rs b/apollo-router/tests/common.rs index 7db5ff0eab..f7eb55fb82 100644 --- a/apollo-router/tests/common.rs +++ b/apollo-router/tests/common.rs @@ -162,7 +162,17 @@ impl IntegrationTest { #[allow(dead_code)] pub async fn start(&mut self) { - let mut router = Command::new(&self.router_location) + let mut router = Command::new(&self.router_location); + if let (Ok(apollo_key), Ok(apollo_graph_ref)) = ( + std::env::var("TEST_APOLLO_KEY"), + std::env::var("TEST_APOLLO_GRAPH_REF"), + ) { + router + .env("APOLLO_KEY", apollo_key) + .env("APOLLO_GRAPH_REF", apollo_graph_ref); + } + + router .args([ "--hr", "--config", @@ -173,9 +183,9 @@ impl IntegrationTest { "--log", "error,apollo_router=info", ]) - .stdout(Stdio::piped()) - .spawn() - .expect("router should start"); + .stdout(Stdio::piped()); + + let mut router = router.spawn().expect("router should start"); let reader = BufReader::new(router.stdout.take().expect("out")); let stdio_tx = self.stdio_tx.clone(); let collect_stdio = self.collect_stdio.take(); diff --git a/apollo-router/tests/fixtures/subscription.router.yaml b/apollo-router/tests/fixtures/subscription.router.yaml index 807bd2661d..2ed382d329 100644 --- a/apollo-router/tests/fixtures/subscription.router.yaml +++ b/apollo-router/tests/fixtures/subscription.router.yaml @@ -7,7 +7,7 @@ homepage: sandbox: enabled: true override_subgraph_url: - accounts: "http://localhost:4041" + products: http://localhost:4005 include_subgraph_errors: all: true subscription: diff --git a/apollo-router/tests/subscription_load_test.rs b/apollo-router/tests/subscription_load_test.rs index d8e8f3d3bc..a59d97c291 100644 --- a/apollo-router/tests/subscription_load_test.rs +++ b/apollo-router/tests/subscription_load_test.rs @@ -10,9 +10,29 @@ use crate::common::Telemetry; mod common; const SUBSCRIPTION_CONFIG: &str = include_str!("fixtures/subscription.router.yaml"); -const SUB_QUERY: &str = r#"subscription { userWasCreated { name reviews { body } }}"#; +const SUB_QUERY: &str = + r#"subscription { userWasCreated(intervalMs: 5, nbEvents: 10) { name reviews { body } }}"#; const UNFEDERATED_SUB_QUERY: &str = r#"subscription { userWasCreated { name username }}"#; +#[tokio::test(flavor = "multi_thread")] +async fn test_subscription() -> Result<(), BoxError> { + let mut router = create_router(SUBSCRIPTION_CONFIG).await?; + router.start().await; + router.assert_started().await; + + let (_, response) = router.run_subscription(SUB_QUERY).await; + assert!(response.status().is_success()); + + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.unwrap(); + assert!(chunk.starts_with(b"\r\n--graphql\r\ncontent-type: application/json\r\n\r\n")); + assert!(chunk.ends_with(b"\r\n--graphql--\r\n")); + } + + Ok(()) +} + #[ignore] #[tokio::test(flavor = "multi_thread")] async fn test_subscription_load() -> Result<(), BoxError> { From 7e074d611e4ebaa0a336dd00180ef2cc9ab7d9a7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 18:55:21 +0000 Subject: [PATCH 05/51] fix(deps): update rust crate clap to 4.4.3 --- Cargo.lock | 10 +++++----- apollo-router-scaffold/Cargo.toml | 2 +- apollo-router/Cargo.toml | 2 +- xtask/Cargo.lock | 4 ++-- xtask/Cargo.toml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d20ce4600..126e198475 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ dependencies = [ "buildstructor 0.5.3", "bytes", "ci_info", - "clap 4.4.2", + "clap 4.4.3", "console-subscriber", "dashmap", "derivative", @@ -429,7 +429,7 @@ version = "1.29.1" dependencies = [ "anyhow", "cargo-scaffold", - "clap 4.4.2", + "clap 4.4.3", "copy_dir", "regex", "str_inflector", @@ -1346,9 +1346,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" dependencies = [ "clap_builder", "clap_derive", @@ -1666,7 +1666,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.2", + "clap 4.4.3", "criterion-plot", "futures", "is-terminal", diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml index 9a6031ec2d..2d35f7e301 100644 --- a/apollo-router-scaffold/Cargo.toml +++ b/apollo-router-scaffold/Cargo.toml @@ -8,7 +8,7 @@ publish = false [dependencies] anyhow = "1.0.75" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } cargo-scaffold = { version = "0.8.10", default-features = false } regex = "1" str_inflector = "0.12.0" diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 8af25ed3df..ce0527d9fa 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -69,7 +69,7 @@ base64 = "0.21.2" bloomfilter = "1.0.12" buildstructor = "0.5.3" bytes = "1.5.0" -clap = { version = "4.4.2", default-features = false, features = [ +clap = { version = "4.4.3", default-features = false, features = [ "env", "derive", "std", diff --git a/xtask/Cargo.lock b/xtask/Cargo.lock index 6bd8d563b2..188b868d76 100644 --- a/xtask/Cargo.lock +++ b/xtask/Cargo.lock @@ -228,9 +228,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" dependencies = [ "clap_builder", "clap_derive", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 90412259fe..7e8f6b3c77 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -11,7 +11,7 @@ publish = false [dependencies] anyhow = "1" camino = "1" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } cargo_metadata = "0.17" # Only use the `clock` features of `chrono` to avoid the `time` dependency # impacted by CVE-2020-26235. https://github.com/chronotope/chrono/issues/602 From cc98b64580c4f70773f882e18e010c255c23492e Mon Sep 17 00:00:00 2001 From: scottmace Date: Wed, 13 Sep 2023 01:44:31 -0700 Subject: [PATCH 06/51] Add Support for AWS X-Ray Tracing Propagator (#3580) Adds support for the OpenTelemetry AWS X-Ray tracing [propagator](https://docs.rs/opentelemetry-aws/latest/opentelemetry_aws/). This propagator helps propagate tracing information from upstream services (such as AWS load balancers) to downstream services and handles conversion between the X-Ray trace id format and OpenTelemetry span contexts. --- .changesets/feat_sm_add_awsxray_propagator.md | 5 +++++ Cargo.lock | 11 +++++++++++ apollo-router/Cargo.toml | 1 + ...uter__configuration__tests__schema_generation.snap | 5 +++++ apollo-router/src/plugins/telemetry/config.rs | 2 ++ apollo-router/src/plugins/telemetry/mod.rs | 3 +++ docs/source/configuration/tracing.mdx | 3 +++ 7 files changed, 30 insertions(+) create mode 100644 .changesets/feat_sm_add_awsxray_propagator.md diff --git a/.changesets/feat_sm_add_awsxray_propagator.md b/.changesets/feat_sm_add_awsxray_propagator.md new file mode 100644 index 0000000000..9b1ff8b2e7 --- /dev/null +++ b/.changesets/feat_sm_add_awsxray_propagator.md @@ -0,0 +1,5 @@ +Adds support for the OpenTelemetry AWS X-Ray tracing propagator. + +This propagator helps propagate tracing information from upstream services (such as AWS load balancers) to downstream services and handles conversion between the X-Ray trace id format and OpenTelemetry span contexts. + +By [@scottmace](https://github.com/scottmace) in https://github.com/apollographql/router/pull/3580 diff --git a/Cargo.lock b/Cargo.lock index 126e198475..40330fd85a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,6 +332,7 @@ dependencies = [ "nu-ansi-term 0.49.0", "once_cell", "opentelemetry", + "opentelemetry-aws", "opentelemetry-datadog", "opentelemetry-http", "opentelemetry-jaeger", @@ -4080,6 +4081,16 @@ dependencies = [ "opentelemetry_sdk", ] +[[package]] +name = "opentelemetry-aws" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a394d24777936802edd6c03a68daab4db39630418c7e431a5648e9befa80b8" +dependencies = [ + "once_cell", + "opentelemetry", +] + [[package]] name = "opentelemetry-datadog" version = "0.7.0" diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index ce0527d9fa..44f7f10234 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -135,6 +135,7 @@ once_cell = "1.18.0" # there (and on `tracing` packages below) should be updated should this change. opentelemetry = { version = "0.19.0", features = ["rt-tokio", "metrics"] } opentelemetry_api = "0.19.0" +opentelemetry-aws = "0.7.0" opentelemetry-datadog = { version = "0.7.0", features = ["reqwest-client"] } opentelemetry-http = "0.8.0" opentelemetry-jaeger = { version = "0.18.0", features = [ diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 14ab198d54..f666bd1baf 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -4993,6 +4993,11 @@ expression: "&schema" "description": "Propagation configuration", "type": "object", "properties": { + "awsxray": { + "description": "Propagate AWS X-Ray", + "default": false, + "type": "boolean" + }, "baggage": { "description": "Propagate baggage https://www.w3.org/TR/baggage/", "default": false, diff --git a/apollo-router/src/plugins/telemetry/config.rs b/apollo-router/src/plugins/telemetry/config.rs index 7d9418dee6..fca3b8c9cf 100644 --- a/apollo-router/src/plugins/telemetry/config.rs +++ b/apollo-router/src/plugins/telemetry/config.rs @@ -337,6 +337,8 @@ pub(crate) struct Propagation { pub(crate) datadog: bool, /// Propagate Zipkin pub(crate) zipkin: bool, + /// Propagate AWS X-Ray + pub(crate) awsxray: bool, } #[derive(Clone, Debug, Deserialize, JsonSchema, Default)] diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index 67450787e9..1c792e3159 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -647,6 +647,9 @@ impl Telemetry { if propagation.datadog || tracing.datadog.is_some() { propagators.push(Box::::default()); } + if propagation.awsxray { + propagators.push(Box::::default()); + } if let Some(from_request_header) = &propagation.request.header_name { propagators.push(Box::new(CustomTraceIdPropagator::new( from_request_header.to_string(), diff --git a/docs/source/configuration/tracing.mdx b/docs/source/configuration/tracing.mdx index 26b294def7..b433e138b2 100644 --- a/docs/source/configuration/tracing.mdx +++ b/docs/source/configuration/tracing.mdx @@ -81,6 +81,9 @@ telemetry: # https://zipkin.io/ (compliant with opentracing) zipkin: false + # https://aws.amazon.com/xray/ (compliant with opentracing) + awsxray: false + # If you have your own way to generate a trace id and you want to pass it via a custom request header request: header_name: my-trace-id From 6648315883c9fed42373d258f7e593fe5e4bd91c Mon Sep 17 00:00:00 2001 From: Jeremy Lempereur Date: Wed, 13 Sep 2023 10:52:56 +0200 Subject: [PATCH 07/51] Coprocessors: Discard content-length sent by coprocessors. (#3802) ControlFlow::Break in coprocessors would lead to all response headers being directly forwarded back to the router, which would lead to content-length missmatch. This changeset discards content-length sent by the coprocessor. --------- Co-authored-by: Coenen Benjamin Co-authored-by: Gary Pennington --- .../fix_igni_coprocessors_discard_content_length.md | 6 ++++++ apollo-router/src/plugins/coprocessor.rs | 11 ++++++++--- apollo-router/src/plugins/coprocessor_test.rs | 3 +++ docs/source/customizations/coprocessor.mdx | 1 + 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 .changesets/fix_igni_coprocessors_discard_content_length.md diff --git a/.changesets/fix_igni_coprocessors_discard_content_length.md b/.changesets/fix_igni_coprocessors_discard_content_length.md new file mode 100644 index 0000000000..3655239e87 --- /dev/null +++ b/.changesets/fix_igni_coprocessors_discard_content_length.md @@ -0,0 +1,6 @@ +### Coprocessors: Discard content-length sent by coprocessors. ([PR #3802](https://github.com/apollographql/router/pull/3802)) + +The `content-length` of an HTTP response can only be computed when a router response is being sent. +We now discard coprocessors `content-length` header to make sure the value is computed correctly. + +By [@o0Ignition0o](https://github.com/o0Ignition0o) in https://github.com/apollographql/router/pull/3802 diff --git a/apollo-router/src/plugins/coprocessor.rs b/apollo-router/src/plugins/coprocessor.rs index fdad623cf8..151d9fa2f5 100644 --- a/apollo-router/src/plugins/coprocessor.rs +++ b/apollo-router/src/plugins/coprocessor.rs @@ -12,8 +12,9 @@ use futures::future::ready; use futures::stream::once; use futures::StreamExt; use futures::TryStreamExt; -use http::header::HeaderName; +use http::header; use http::HeaderMap; +use http::HeaderName; use http::HeaderValue; use hyper::client::HttpConnector; use hyper::Body; @@ -1114,8 +1115,12 @@ pub(super) fn externalize_header_map( pub(super) fn internalize_header_map( input: HashMap>, ) -> Result, BoxError> { - let mut output = HeaderMap::new(); - for (k, values) in input { + // better than nothing even though it doesnt account for the values len + let mut output = HeaderMap::with_capacity(input.len()); + for (k, values) in input + .into_iter() + .filter(|(k, _)| k != header::CONTENT_LENGTH.as_str()) + { for v in values { let key = HeaderName::from_str(k.as_ref())?; let value = HeaderValue::from_str(v.as_ref())?; diff --git a/apollo-router/src/plugins/coprocessor_test.rs b/apollo-router/src/plugins/coprocessor_test.rs index 26ef229eb7..76d4177768 100644 --- a/apollo-router/src/plugins/coprocessor_test.rs +++ b/apollo-router/src/plugins/coprocessor_test.rs @@ -1159,6 +1159,9 @@ mod tests { ], ); + // This header should be stripped + external_form.insert("content-length".to_string(), vec!["1024".to_string()]); + let actual = internalize_header_map(external_form).expect("internalized header map"); assert_eq!(expected, actual); diff --git a/docs/source/customizations/coprocessor.mdx b/docs/source/customizations/coprocessor.mdx index 5f02be67c3..b01cf4e301 100644 --- a/docs/source/customizations/coprocessor.mdx +++ b/docs/source/customizations/coprocessor.mdx @@ -636,6 +636,7 @@ If your coprocessor [returns a _different_ value](#responding-to-coprocessor-req An object mapping of all HTTP header names and values for the corresponding request or response. If your coprocessor [returns a _different_ value](#responding-to-coprocessor-requests) for `headers`, the router replaces the existing headers with that value. +⚠️ `content-length` sent by Coprocessors are discarded by the router. (Wrong `content-length` values could lead to failure for HTTP requests to be sent) From 36cd258584376491517f850bf82e7203080241a8 Mon Sep 17 00:00:00 2001 From: Gary Pennington Date: Wed, 13 Sep 2023 09:54:42 +0100 Subject: [PATCH 08/51] Rhai documentation: remove incorrect statement about request.subgraph fields (#3808) It is possible to modify `request.subgraph` fields from a `Rhai` script and so the documentation should reflect that. **Checklist** Complete the checklist (and note appropriate exceptions) before a final PR is raised. - [ ] Changes are compatible[^1] - [ ] Documentation[^2] completed - [ ] Performance impact assessed and acceptable - Tests added and passing[^3] - [ ] Unit Tests - [ ] Integration Tests - [ ] Manual Tests **Exceptions** *Note any exceptions here* **Notes** [^1]. It may be appropriate to bring upcoming changes to the attention of other (impacted) groups. Please endeavour to do this before seeking PR approval. The mechanism for doing this will vary considerably, so use your judgement as to how and when to do this. [^2]. Configuration is an important part of many changes. Where applicable please try to document configuration examples. [^3]. Tick whichever testing boxes are applicable. If you are adding Manual Tests: - please document the manual testing (extensively) in the Exceptions. - please raise a separate issue to automate the test and label it (or ask for it to be labeled) as `manual test` --- .changesets/docs_garypen_fix_rhai_subgraph_docs.md | 5 +++++ docs/source/customizations/rhai-api.mdx | 8 +++----- 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 .changesets/docs_garypen_fix_rhai_subgraph_docs.md diff --git a/.changesets/docs_garypen_fix_rhai_subgraph_docs.md b/.changesets/docs_garypen_fix_rhai_subgraph_docs.md new file mode 100644 index 0000000000..94f696f5fa --- /dev/null +++ b/.changesets/docs_garypen_fix_rhai_subgraph_docs.md @@ -0,0 +1,5 @@ +### Rhai documentation: remove incorrect statement about request.subgraph fields ([PR #3808](https://github.com/apollographql/router/pull/3808)) + +It is possible to modify `request.subgraph` fields from a `Rhai` script and so the documentation should reflect that. + +By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3808 \ No newline at end of file diff --git a/docs/source/customizations/rhai-api.mdx b/docs/source/customizations/rhai-api.mdx index 83775e96e5..add590aefb 100644 --- a/docs/source/customizations/rhai-api.mdx +++ b/docs/source/customizations/rhai-api.mdx @@ -289,7 +289,7 @@ Note: get() may fail, so it's best to handle exceptions when using it. ## `Request` interface -All callback functions registered via `map_request` are passed a `request` object that represents the request sent by the client. This object provides the following fields, most of which a callback can modify in-place (read-write): +All callback functions registered via `map_request` are passed a `request` object that represents the request sent by the client. This object provides the following fields: ``` request.context @@ -302,9 +302,9 @@ request.body.extensions request.uri.host request.uri.path ``` -Note: `method` is read-only. +Note: These fields are typically modifiable, apart from `method` which is always read-only. However, when the callback service is `subgraph_service`,the only modifiable field is `request.context`. -**For `subgraph_service` callbacks only,** the `request` object provides _additional_ fields for interacting with the request that will be sent to the corresponding subgraph: +**For `subgraph_service` callbacks only,** the `request` object provides _additional_ modifiable fields for interacting with the request that will be sent to the corresponding subgraph: ``` request.subgraph.headers @@ -316,8 +316,6 @@ request.subgraph.uri.host request.subgraph.uri.path ``` -These additional fields are shared across all subgraph invocations and are thus read-only. - ### `request.context` The context is a generic key/value store that exists for the entire lifespan of a particular client request. You can use this to share information between multiple callbacks throughout the request's lifespan. From e496f747ca9ffbbb7e6e82814c86db63ef897db6 Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Wed, 13 Sep 2023 11:28:23 +0200 Subject: [PATCH 09/51] do not run test_subscription if graph ref and apollo key is not set for tests (#3813) Signed-off-by: Benjamin Coenen <5719034+bnjjj@users.noreply.github.com> --- apollo-router/tests/subscription_load_test.rs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/apollo-router/tests/subscription_load_test.rs b/apollo-router/tests/subscription_load_test.rs index a59d97c291..99f72ff0e1 100644 --- a/apollo-router/tests/subscription_load_test.rs +++ b/apollo-router/tests/subscription_load_test.rs @@ -16,18 +16,20 @@ const UNFEDERATED_SUB_QUERY: &str = r#"subscription { userWasCreated { name use #[tokio::test(flavor = "multi_thread")] async fn test_subscription() -> Result<(), BoxError> { - let mut router = create_router(SUBSCRIPTION_CONFIG).await?; - router.start().await; - router.assert_started().await; + if std::env::var("TEST_APOLLO_KEY").is_ok() && std::env::var("TEST_APOLLO_GRAPH_REF").is_ok() { + let mut router = create_router(SUBSCRIPTION_CONFIG).await?; + router.start().await; + router.assert_started().await; - let (_, response) = router.run_subscription(SUB_QUERY).await; - assert!(response.status().is_success()); + let (_, response) = router.run_subscription(SUB_QUERY).await; + assert!(response.status().is_success()); - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.unwrap(); - assert!(chunk.starts_with(b"\r\n--graphql\r\ncontent-type: application/json\r\n\r\n")); - assert!(chunk.ends_with(b"\r\n--graphql--\r\n")); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.unwrap(); + assert!(chunk.starts_with(b"\r\n--graphql\r\ncontent-type: application/json\r\n\r\n")); + assert!(chunk.ends_with(b"\r\n--graphql--\r\n")); + } } Ok(()) From 43ddab8b383ec7b6f70cd407ef4dff544cf11c87 Mon Sep 17 00:00:00 2001 From: Jesse Rosenberger Date: Wed, 13 Sep 2023 16:55:12 +0300 Subject: [PATCH 10/51] chore: Fix formatting in `pull_request_template.md` (#3814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This has looked pretty funny to me for a while now and the fix was as simple as using `:` instead of `.`! Inspired by @nathanmarcos doing it as a one-off in https://github.com/apollographql/router/pull/3805. 😸 I also took the opportunity to add a nice horizontal rule that improves the ascetics of the rendered result and made some other small tweaks to text/formatting. Most notably though, I removed the suggestion to open a issue labeled `manual test`. It takes up a lot of real-estate in the text, and not a single issue has been opened or labeled as such. (Nor does the label even exist. 😅 ) --- .github/pull_request_template.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a01e50f728..3de4040763 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,13 @@ *Description here* -Fixes #issue_number +Fixes #**issue_number** +--- **Checklist** -Complete the checklist (and note appropriate exceptions) before a final PR is raised. +Complete the checklist (and note appropriate exceptions) before the PR is marked ready-for-review. - [ ] Changes are compatible[^1] - [ ] Documentation[^2] completed @@ -22,8 +23,6 @@ Complete the checklist (and note appropriate exceptions) before a final PR is ra **Notes** -[^1]. It may be appropriate to bring upcoming changes to the attention of other (impacted) groups. Please endeavour to do this before seeking PR approval. The mechanism for doing this will vary considerably, so use your judgement as to how and when to do this. -[^2]. Configuration is an important part of many changes. Where applicable please try to document configuration examples. -[^3]. Tick whichever testing boxes are applicable. If you are adding Manual Tests: - - please document the manual testing (extensively) in the Exceptions. - - please raise a separate issue to automate the test and label it (or ask for it to be labeled) as `manual test` +[^1]: It may be appropriate to bring upcoming changes to the attention of other (impacted) groups. Please endeavour to do this before seeking PR approval. The mechanism for doing this will vary considerably, so use your judgement as to how and when to do this. +[^2]: Configuration is an important part of many changes. Where applicable please try to document configuration examples. +[^3]: Tick whichever testing boxes are applicable. If you are adding Manual Tests, please document the manual testing (extensively) in the Exceptions. From 9ae60c12c0c3860dab373ddf78f79da3993c063e Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Wed, 13 Sep 2023 17:02:09 +0200 Subject: [PATCH 11/51] chore(xtask): fix changeset create with the new pr template (#3818) Fix a regression introduced in #3814 --------- Signed-off-by: Benjamin Coenen <5719034+bnjjj@users.noreply.github.com> --- xtask/src/commands/changeset/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtask/src/commands/changeset/mod.rs b/xtask/src/commands/changeset/mod.rs index 7ed7b5950a..43203a56b0 100644 --- a/xtask/src/commands/changeset/mod.rs +++ b/xtask/src/commands/changeset/mod.rs @@ -395,7 +395,7 @@ impl Create { // In the future, we will use the "start metadata" HTML tag, but for now, // we support both. let pr_body_trailer_regex = regex::Regex::new( - r"(?ms)(^\n$\n)?^\*\*Checklist\*\*$[\s\S]*", + r"(?ms)(^\n---$\n)?^\*\*Checklist\*\*$[\s\S]*", )?; // Remove all the "Fixes" references, since we're already going to reference From 8a2774b1ad5472fb9d5bd96abe537a4ffd4d2763 Mon Sep 17 00:00:00 2001 From: Matt Peake <7741049+peakematt@users.noreply.github.com> Date: Wed, 13 Sep 2023 11:43:26 -0400 Subject: [PATCH 12/51] Additional benign positive allowlisting (#3816) This implements additional allowlisting for benign positive "secret" detection. The values allowlisted in this PR are test/sample JWTs and private keys. Apologies for repeated PRs on this. These detections appeared after my last PR for allowlisting secrets. I believe that gitleaks (the project we're using to scan for secrets) updated their signatures to better detect JWTs, causing more detections. I've run gitleaks on this repo by hand and after this change, there are currently no additional detections generated. **Checklist** Complete the checklist (and note appropriate exceptions) before a final PR is raised. - [ ] Changes are compatible[^1] - [ ] Documentation[^2] completed - [ ] Performance impact assessed and acceptable - Tests added and passing[^3] - [ ] Unit Tests - [ ] Integration Tests - [ ] Manual Tests **Exceptions** *Note any exceptions here* **Notes** [^1]. It may be appropriate to bring upcoming changes to the attention of other (impacted) groups. Please endeavour to do this before seeking PR approval. The mechanism for doing this will vary considerably, so use your judgement as to how and when to do this. [^2]. Configuration is an important part of many changes. Where applicable please try to document configuration examples. [^3]. Tick whichever testing boxes are applicable. If you are adding Manual Tests: - please document the manual testing (extensively) in the Exceptions. - please raise a separate issue to automate the test and label it (or ask for it to be labeled) as `manual test` --- .gitleaks.toml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitleaks.toml b/.gitleaks.toml index 2fd8c14e8f..44872617d2 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -14,17 +14,25 @@ [ rules.allowlist ] commits = [ "7fdb0a5a0831d309524e98503760c16bd3de160c", - # Example private key for test cases + # Example private keys for test cases "9d6464b96eed3a20c48d5ec653bf9986757c89e5", - + "74ebb0269933c703efd9143c029f69e30330478b", ] [[ rules ]] id = "jwt" [ rules.allowlist ] commits = [ - # Example jwt + # Example jwts "f12bcddc663aa4c4d90218a1bb718fe74e0e7be3", + "6ff43869e0cf8ab0de8dd0e898f80c08ad135552", + "b11412918d58db34693488b29495e148dbb27cc2", + "a4ceefc8d51879330f10861438401a3a3deb316e", + "d19de3cbca36981676ff03c7f3cab113609772b1", + "3ee3d4bda9efcf4f175fdcfd59abc16cb9a4657e", + "76aaa5ec21d4fbde65676c6cf0dc265dd6daee99", + "d2b85acee5cf94120b47093f87ae80a7365d7b28", + "f107c2a42292d0c8068395f5cb8f491377559a42", ] [[ rules ]] From ffa1d12974246dddc6d0b52ff441921e6530c792 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 00:15:26 +0000 Subject: [PATCH 13/51] fix(deps): update rust crate serde_json to 1.0.107 --- Cargo.lock | 4 ++-- apollo-router/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40330fd85a..ebe2b00d53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5591,9 +5591,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "indexmap 2.0.0", "itoa", diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 44f7f10234..817cf8f255 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -183,7 +183,7 @@ shellexpand = "3.1.0" sha2 = "0.10.7" serde = { version = "1.0.188", features = ["derive", "rc"] } serde_json_bytes = { version = "0.2.1", features = ["preserve_order"] } -serde_json = { version = "1.0.106", features = [ +serde_json = { version = "1.0.107", features = [ "preserve_order", "float_roundtrip", ] } From cf19df6b744043f3b60f55b86a75a3b26d0fd610 Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Thu, 14 Sep 2023 09:59:53 +0200 Subject: [PATCH 14/51] Supergraph coprocessor implementation (#3647) Co-authored-by: o0Ignition0o --- .../feat_geal_supergraph_coprocessor2.md | 33 + CHANGELOG.md | 10 +- ...test__metrics@coprocessor.router.yaml.snap | 4 +- ...nfiguration__tests__schema_generation.snap | 100 ++ .../testdata/metrics/coprocessor.router.yaml | 13 + .../{coprocessor.rs => coprocessor/mod.rs} | 40 +- .../src/plugins/coprocessor/supergraph.rs | 897 ++++++++++++++++++ .../test.rs} | 2 +- apollo-router/src/plugins/mod.rs | 2 - apollo-router/src/services/external.rs | 36 + docs/source/customizations/coprocessor.mdx | 15 +- 11 files changed, 1134 insertions(+), 18 deletions(-) create mode 100644 .changesets/feat_geal_supergraph_coprocessor2.md rename apollo-router/src/plugins/{coprocessor.rs => coprocessor/mod.rs} (97%) create mode 100644 apollo-router/src/plugins/coprocessor/supergraph.rs rename apollo-router/src/plugins/{coprocessor_test.rs => coprocessor/test.rs} (99%) diff --git a/.changesets/feat_geal_supergraph_coprocessor2.md b/.changesets/feat_geal_supergraph_coprocessor2.md new file mode 100644 index 0000000000..9dd3030a34 --- /dev/null +++ b/.changesets/feat_geal_supergraph_coprocessor2.md @@ -0,0 +1,33 @@ +### Supergraph coprocessor implementation ([PR #3647](https://github.com/apollographql/router/pull/3647)) + +Coprocessors now support supergraph service interception. + +On the request side, the coprocessor payload can contain: +- method +- headers +- body +- context +- sdl + +On the response side, the payload can contain: +- status_code +- headers +- body +- context +- sdl + +The supergraph request body contains: +* query +* operation name +* variables +* extensions + +The supergraph response body contains: +* label +* data +* errors +* extensions + +When using `@defer` or subscriptions a supergraph response may contain multiple GraphQL responses, and the coprocessor will be called for each. + +By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3647 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index e3050a84b1..082c5f4c08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -364,7 +364,7 @@ By [@garypen](https://github.com/garypen) in https://github.com/apollographql/ro ### Spelling of `content_negociation` corrected to `content_negotiation` ([Issue #3204](https://github.com/apollographql/router/issues/3204)) -We had a bit of a French twist on one of our internal module names. We won't promise it won't happen again, but `content_negociation` is spelled as `content_negotiation` now. 😄 +We had a bit of a French twist on one of our internal module names. We won't promise it won't happen again, but `content_negociation` is spelled as `content_negotiation` now. 😄 Thank you for this contribution! @@ -1720,7 +1720,7 @@ By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/p ### Small gzip'd responses no longer cause a panic -A regression introduced in v1.17.0 — again related to compression — has been resolved. This occurred when small responses used invalid buffer management, causing a panic. +A regression introduced in v1.17.0 — again related to compression — has been resolved. This occurred when small responses used invalid buffer management, causing a panic. By [@dbanty](https://github.com/dbanty) in https://github.com/apollographql/router/pull/3047 @@ -2254,7 +2254,7 @@ By [@BrynCooke](https://github.com/BrynCooke) in https://github.com/apollographq ### Distributed caching: Don't send Redis' `CLIENT SETNAME` ([PR #2825](https://github.com/apollographql/router/pull/2825)) -We won't send [the `CLIENT SETNAME` command](https://redis.io/commands/client-setname/) to connected Redis servers. This resolves an incompatibility with some Redis-compatible servers since not all "Redis-compatible" offerings (like Google Memorystore) actually support _every_ Redis command. We weren't actually necessitating this feature, it was just a feature that could be enabled optionally on our Redis client. No Router functionality is impacted. +We won't send [the `CLIENT SETNAME` command](https://redis.io/commands/client-setname/) to connected Redis servers. This resolves an incompatibility with some Redis-compatible servers since not all "Redis-compatible" offerings (like Google Memorystore) actually support _every_ Redis command. We weren't actually necessitating this feature, it was just a feature that could be enabled optionally on our Redis client. No Router functionality is impacted. By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/2825 @@ -3424,7 +3424,7 @@ By [@Geal](https://github.com/geal) in https://github.com/apollographql/router/p ### Optimize header propagation plugin's regular expression matching ([PR #2392](https://github.com/apollographql/router/pull/2392)) -We've changed the header propagation plugins' behavior to reduce the chance of memory allocations occurring when applying regex-based header propagation rules. +We've changed the header propagation plugins' behavior to reduce the chance of memory allocations occurring when applying regex-based header propagation rules. By [@o0Ignition0o](https://github.com/o0Ignition0o) in https://github.com/apollographql/router/pull/2392 @@ -3474,7 +3474,7 @@ By [@Geal](https://github.com/geal) in https://github.com/apollographql/router/p Configuration changes will be [automatically migrated on load](https://www.apollographql.com/docs/router/configuration/overview#upgrading-your-router-configuration). However, you should update your source configuration files as these will become breaking changes in a future major release. -### Defer support graduates from preview ([Issue #2368](https://github.com/apollographql/router/issues/2368)) +### Defer support graduates from preview ([Issue #2368](https://github.com/apollographql/router/issues/2368)) We're pleased to announce that [`@defer` support](https://www.apollographql.com/docs/router/executing-operations/defer-support/) has been promoted to general availability in accordance with our [product launch stages](https://www.apollographql.com/docs/resources/product-launch-stages/). diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@coprocessor.router.yaml.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@coprocessor.router.yaml.snap index bdc1a7899b..4c2078200a 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@coprocessor.router.yaml.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@coprocessor.router.yaml.snap @@ -8,6 +8,6 @@ value.apollo.router.config.coprocessor: opt__router__response__: true opt__subgraph__request__: true opt__subgraph__response__: true - opt__supergraph__request__: false - opt__supergraph__response__: false + opt__supergraph__request__: true + opt__supergraph__response__: true diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index f666bd1baf..a175614292 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -779,6 +779,106 @@ expression: "&schema" }, "additionalProperties": false }, + "supergraph": { + "description": "The supergraph stage request/response configuration", + "default": { + "request": { + "headers": false, + "context": false, + "body": false, + "sdl": false, + "method": false + }, + "response": { + "headers": false, + "context": false, + "body": false, + "sdl": false, + "status_code": false + } + }, + "type": "object", + "properties": { + "request": { + "description": "The request configuration", + "default": { + "headers": false, + "context": false, + "body": false, + "sdl": false, + "method": false + }, + "type": "object", + "properties": { + "body": { + "description": "Send the body", + "default": false, + "type": "boolean" + }, + "context": { + "description": "Send the context", + "default": false, + "type": "boolean" + }, + "headers": { + "description": "Send the headers", + "default": false, + "type": "boolean" + }, + "method": { + "description": "Send the method", + "default": false, + "type": "boolean" + }, + "sdl": { + "description": "Send the SDL", + "default": false, + "type": "boolean" + } + }, + "additionalProperties": false + }, + "response": { + "description": "What information is passed to a router request/response stage", + "default": { + "headers": false, + "context": false, + "body": false, + "sdl": false, + "status_code": false + }, + "type": "object", + "properties": { + "body": { + "description": "Send the body", + "default": false, + "type": "boolean" + }, + "context": { + "description": "Send the context", + "default": false, + "type": "boolean" + }, + "headers": { + "description": "Send the headers", + "default": false, + "type": "boolean" + }, + "sdl": { + "description": "Send the SDL", + "default": false, + "type": "boolean" + }, + "status_code": { + "description": "Send the HTTP status", + "default": false, + "type": "boolean" + } + }, + "additionalProperties": false + } + } + }, "timeout": { "description": "The timeout for external requests", "default": { diff --git a/apollo-router/src/configuration/testdata/metrics/coprocessor.router.yaml b/apollo-router/src/configuration/testdata/metrics/coprocessor.router.yaml index a1a7714784..12ad67acf7 100644 --- a/apollo-router/src/configuration/testdata/metrics/coprocessor.router.yaml +++ b/apollo-router/src/configuration/testdata/metrics/coprocessor.router.yaml @@ -15,6 +15,19 @@ coprocessor: body: true headers: true status_code: true + supergraph: + request: + headers: true + body: true + context: true + method: true + sdl: true + response: + sdl: true + context: true + body: true + headers: true + status_code: true subgraph: all: request: diff --git a/apollo-router/src/plugins/coprocessor.rs b/apollo-router/src/plugins/coprocessor/mod.rs similarity index 97% rename from apollo-router/src/plugins/coprocessor.rs rename to apollo-router/src/plugins/coprocessor/mod.rs index 151d9fa2f5..5580c3d2da 100644 --- a/apollo-router/src/plugins/coprocessor.rs +++ b/apollo-router/src/plugins/coprocessor/mod.rs @@ -36,6 +36,7 @@ use crate::layers::ServiceBuilderExt; use crate::plugin::Plugin; use crate::plugin::PluginInit; use crate::register_plugin; +use crate::services; use crate::services::external::Control; use crate::services::external::Externalizable; use crate::services::external::PipelineStep; @@ -45,6 +46,11 @@ use crate::services::router; use crate::services::subgraph; use crate::tracer::TraceId; +#[cfg(test)] +mod test; + +mod supergraph; + pub(crate) const EXTERNAL_SPAN_NAME: &str = "external_plugin"; const POOL_IDLE_TIMEOUT_DURATION: Option = Some(Duration::from_secs(5)); @@ -87,6 +93,13 @@ impl Plugin for CoprocessorPlugin { self.router_service(service) } + fn supergraph_service( + &self, + service: services::supergraph::BoxService, + ) -> services::supergraph::BoxService { + self.supergraph_service(service) + } + fn subgraph_service(&self, name: &str, service: subgraph::BoxService) -> subgraph::BoxService { self.subgraph_service(name, service) } @@ -150,6 +163,18 @@ where ) } + fn supergraph_service( + &self, + service: services::supergraph::BoxService, + ) -> services::supergraph::BoxService { + self.configuration.supergraph.as_service( + self.http_client.clone(), + service, + self.configuration.url.clone(), + self.sdl.clone(), + ) + } + fn subgraph_service(&self, name: &str, service: subgraph::BoxService) -> subgraph::BoxService { self.configuration.subgraph.all.as_service( self.http_client.clone(), @@ -240,6 +265,9 @@ struct Conf { /// The router stage request/response configuration #[serde(default)] router: RouterStage, + /// The supergraph stage request/response configuration + #[serde(default)] + supergraph: supergraph::SupergraphStage, /// The subgraph stage request/response configuration #[serde(default)] subgraph: SubgraphStages, @@ -677,7 +705,7 @@ where // If first is None, or contains an error we return an error let opt_first: Option = first.and_then(|f| f.ok()); let bytes = match opt_first { - Some(b) => b.to_vec(), + Some(b) => b, None => { tracing::error!( "Coprocessor cannot convert body into future due to problem with first part" @@ -696,7 +724,7 @@ where .transpose()?; let body_to_send = response_config .body - .then(|| String::from_utf8(bytes.clone())) + .then(|| std::str::from_utf8(&bytes).map(|s| s.to_string())) .transpose()?; let status_to_send = response_config.status_code.then(|| parts.status.as_u16()); let context_to_send = response_config.context.then(|| response.context.clone()); @@ -858,7 +886,6 @@ where // First, extract the data we need from our request and prepare our // external call. Use our configuration to figure out which data to send. let (parts, body) = request.subgraph_request.into_parts(); - let bytes = Bytes::from(serde_json::to_vec(&body)?); let headers_to_send = request_config .headers @@ -867,7 +894,7 @@ where let body_to_send = request_config .body - .then(|| serde_json::from_slice::(&bytes)) + .then(|| serde_json::to_value(&body)) .transpose()?; let context_to_send = request_config.context.then(|| request.context.clone()); let uri = request_config.uri.then(|| parts.uri.to_string()); @@ -998,7 +1025,6 @@ where // external call. Use our configuration to figure out which data to send. let (parts, body) = response.response.into_parts(); - let bytes = Bytes::from(serde_json::to_vec(&body)?); let headers_to_send = response_config .headers @@ -1009,7 +1035,7 @@ where let body_to_send = response_config .body - .then(|| serde_json::from_slice::(&bytes)) + .then(|| serde_json::to_value(&body)) .transpose()?; let context_to_send = response_config.context.then(|| response.context.clone()); let service_name = response_config.service_name.then_some(service_name); @@ -1099,7 +1125,7 @@ fn validate_coprocessor_output( } /// Convert a HeaderMap into a HashMap -pub(super) fn externalize_header_map( +pub(crate) fn externalize_header_map( input: &HeaderMap, ) -> Result>, BoxError> { let mut output = HashMap::new(); diff --git a/apollo-router/src/plugins/coprocessor/supergraph.rs b/apollo-router/src/plugins/coprocessor/supergraph.rs new file mode 100644 index 0000000000..a76de0763e --- /dev/null +++ b/apollo-router/src/plugins/coprocessor/supergraph.rs @@ -0,0 +1,897 @@ +use std::ops::ControlFlow; +use std::sync::Arc; + +use futures::future; +use futures::stream; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use tower::BoxError; +use tower::ServiceBuilder; +use tower_service::Service; + +use super::externalize_header_map; +use super::*; +use crate::graphql; +use crate::layers::async_checkpoint::AsyncCheckpointLayer; +use crate::layers::ServiceBuilderExt; +use crate::plugins::coprocessor::EXTERNAL_SPAN_NAME; +use crate::response; +use crate::services::supergraph; + +/// What information is passed to a router request/response stage +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, JsonSchema)] +#[serde(default, deny_unknown_fields)] +pub(super) struct SupergraphRequestConf { + /// Send the headers + pub(super) headers: bool, + /// Send the context + pub(super) context: bool, + /// Send the body + pub(super) body: bool, + /// Send the SDL + pub(super) sdl: bool, + /// Send the method + pub(super) method: bool, +} + +/// What information is passed to a router request/response stage +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, JsonSchema)] +#[serde(default, deny_unknown_fields)] +pub(super) struct SupergraphResponseConf { + /// Send the headers + pub(super) headers: bool, + /// Send the context + pub(super) context: bool, + /// Send the body + pub(super) body: bool, + /// Send the SDL + pub(super) sdl: bool, + /// Send the HTTP status + pub(super) status_code: bool, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, JsonSchema)] +#[serde(default)] +pub(super) struct SupergraphStage { + /// The request configuration + pub(super) request: SupergraphRequestConf, + // /// The response configuration + pub(super) response: SupergraphResponseConf, +} + +impl SupergraphStage { + pub(crate) fn as_service( + &self, + http_client: C, + service: supergraph::BoxService, + coprocessor_url: String, + sdl: Arc, + ) -> supergraph::BoxService + where + C: Service, Response = hyper::Response, Error = BoxError> + + Clone + + Send + + Sync + + 'static, + >>::Future: Send + 'static, + { + let request_layer = (self.request != Default::default()).then_some({ + let request_config = self.request.clone(); + let coprocessor_url = coprocessor_url.clone(); + let http_client = http_client.clone(); + let sdl = sdl.clone(); + + AsyncCheckpointLayer::new(move |request: supergraph::Request| { + let request_config = request_config.clone(); + let coprocessor_url = coprocessor_url.clone(); + let http_client = http_client.clone(); + let sdl = sdl.clone(); + + async move { + let mut succeeded = true; + let result = process_supergraph_request_stage( + http_client, + coprocessor_url, + sdl, + request, + request_config, + ) + .await + .map_err(|error| { + succeeded = false; + tracing::error!( + "external extensibility: supergraph request stage error: {error}" + ); + error + }); + tracing::info!( + monotonic_counter.apollo.router.operations.coprocessor = 1u64, + coprocessor.stage = %PipelineStep::SupergraphRequest, + coprocessor.succeeded = succeeded, + "Total operations with co-processors enabled" + ); + result + } + }) + }); + + let response_layer = (self.response != Default::default()).then_some({ + let response_config = self.response.clone(); + + MapFutureLayer::new(move |fut| { + let coprocessor_url = coprocessor_url.clone(); + let sdl: Arc = sdl.clone(); + let http_client = http_client.clone(); + let response_config = response_config.clone(); + + async move { + let response: supergraph::Response = fut.await?; + + let mut succeeded = true; + let result = process_supergraph_response_stage( + http_client, + coprocessor_url, + sdl, + response, + response_config, + ) + .await + .map_err(|error| { + succeeded = false; + tracing::error!( + "external extensibility: router response stage error: {error}" + ); + error + }); + tracing::info!( + monotonic_counter.apollo.router.operations.coprocessor = 1u64, + coprocessor.stage = %PipelineStep::SupergraphResponse, + coprocessor.succeeded = succeeded, + "Total operations with co-processors enabled" + ); + result + } + }) + }); + + fn external_service_span() -> impl Fn(&supergraph::Request) -> tracing::Span + Clone { + move |_request: &supergraph::Request| { + tracing::info_span!( + EXTERNAL_SPAN_NAME, + "external service" = stringify!(supergraph::Request), + "otel.kind" = "INTERNAL" + ) + } + } + + ServiceBuilder::new() + .instrument(external_service_span()) + .option_layer(request_layer) + .option_layer(response_layer) + .buffered() + .service(service) + .boxed() + } +} + +async fn process_supergraph_request_stage( + http_client: C, + coprocessor_url: String, + sdl: Arc, + mut request: supergraph::Request, + request_config: SupergraphRequestConf, +) -> Result, BoxError> +where + C: Service, Response = hyper::Response, Error = BoxError> + + Clone + + Send + + Sync + + 'static, + >>::Future: Send + 'static, +{ + // Call into our out of process processor with a body of our body + // First, extract the data we need from our request and prepare our + // external call. Use our configuration to figure out which data to send. + let (parts, body) = request.supergraph_request.into_parts(); + let bytes = Bytes::from(serde_json::to_vec(&body)?); + + let headers_to_send = request_config + .headers + .then(|| externalize_header_map(&parts.headers)) + .transpose()?; + + let body_to_send = request_config + .body + .then(|| serde_json::from_slice::(&bytes)) + .transpose()?; + let context_to_send = request_config.context.then(|| request.context.clone()); + let sdl_to_send = request_config.sdl.then(|| sdl.clone().to_string()); + let method = request_config.method.then(|| parts.method.to_string()); + + let payload = Externalizable::supergraph_builder() + .stage(PipelineStep::SupergraphRequest) + .control(Control::default()) + .and_id(TraceId::maybe_new().map(|id| id.to_string())) + .and_headers(headers_to_send) + .and_body(body_to_send) + .and_context(context_to_send) + .and_method(method) + .and_sdl(sdl_to_send) + .build(); + + tracing::debug!(?payload, "externalized output"); + let guard = request.context.enter_active_request(); + let start = Instant::now(); + let co_processor_result = payload.call(http_client, &coprocessor_url).await; + let duration = start.elapsed().as_secs_f64(); + drop(guard); + tracing::info!( + histogram.apollo.router.operations.coprocessor.duration = duration, + coprocessor.stage = %PipelineStep::SupergraphRequest, + ); + + tracing::debug!(?co_processor_result, "co-processor returned"); + let co_processor_output = co_processor_result?; + validate_coprocessor_output(&co_processor_output, PipelineStep::SupergraphRequest)?; + // unwrap is safe here because validate_coprocessor_output made sure control is available + let control = co_processor_output.control.expect("validated above; qed"); + + // Thirdly, we need to interpret the control flow which may have been + // updated by our co-processor and decide if we should proceed or stop. + + if matches!(control, Control::Break(_)) { + // Ensure the code is a valid http status code + let code = control.get_http_status()?; + + let res = { + let graphql_response: crate::graphql::Response = + serde_json::from_value(co_processor_output.body.unwrap_or(serde_json::Value::Null)) + .unwrap_or_else(|error| { + crate::graphql::Response::builder() + .errors(vec![Error::builder() + .message(format!( + "couldn't deserialize coprocessor output body: {error}" + )) + .extension_code("EXTERNAL_DESERIALIZATION_ERROR") + .build()]) + .build() + }); + + let mut http_response = http::Response::builder() + .status(code) + .body(stream::once(future::ready(graphql_response)).boxed())?; + if let Some(headers) = co_processor_output.headers { + *http_response.headers_mut() = internalize_header_map(headers)?; + } + + let supergraph_response = supergraph::Response { + response: http_response, + context: request.context, + }; + + if let Some(context) = co_processor_output.context { + for (key, value) in context.try_into_iter()? { + supergraph_response + .context + .upsert_json_value(key, move |_current| value); + } + } + + supergraph_response + }; + return Ok(ControlFlow::Break(res)); + } + + // Finally, process our reply and act on the contents. Our processing logic is + // that we replace "bits" of our incoming request with the updated bits if they + // are present in our co_processor_output. + + let new_body: crate::graphql::Request = match co_processor_output.body { + Some(value) => serde_json::from_value(value)?, + None => body, + }; + + request.supergraph_request = http::Request::from_parts(parts, new_body); + + if let Some(context) = co_processor_output.context { + for (key, value) in context.try_into_iter()? { + request + .context + .upsert_json_value(key, move |_current| value); + } + } + + if let Some(headers) = co_processor_output.headers { + *request.supergraph_request.headers_mut() = internalize_header_map(headers)?; + } + + if let Some(uri) = co_processor_output.uri { + *request.supergraph_request.uri_mut() = uri.parse()?; + } + + Ok(ControlFlow::Continue(request)) +} + +async fn process_supergraph_response_stage( + http_client: C, + coprocessor_url: String, + sdl: Arc, + response: supergraph::Response, + response_config: SupergraphResponseConf, +) -> Result +where + C: Service, Response = hyper::Response, Error = BoxError> + + Clone + + Send + + Sync + + 'static, + >>::Future: Send + 'static, +{ + // split the response into parts + body + let (mut parts, body) = response.response.into_parts(); + + // we split the body (which is a stream) into first response + rest of responses, + // for which we will implement mapping later + let (first, rest): (Option, graphql::ResponseStream) = + body.into_future().await; + + // If first is None, we return an error + let first = first.ok_or_else(|| { + BoxError::from("Coprocessor cannot convert body into future due to problem with first part") + })?; + + // Now we process our first chunk of response + // Encode headers, body, status, context, sdl to create a payload + let headers_to_send = response_config + .headers + .then(|| externalize_header_map(&parts.headers)) + .transpose()?; + let body_to_send = response_config + .body + .then(|| serde_json::to_value(&first).expect("serialization will not fail")); + let status_to_send = response_config.status_code.then(|| parts.status.as_u16()); + let context_to_send = response_config.context.then(|| response.context.clone()); + let sdl_to_send = response_config.sdl.then(|| sdl.clone().to_string()); + + let payload = Externalizable::supergraph_builder() + .stage(PipelineStep::SupergraphResponse) + .and_id(TraceId::maybe_new().map(|id| id.to_string())) + .and_headers(headers_to_send) + .and_body(body_to_send) + .and_context(context_to_send) + .and_status_code(status_to_send) + .and_sdl(sdl_to_send.clone()) + .build(); + + // Second, call our co-processor and get a reply. + tracing::debug!(?payload, "externalized output"); + let guard = response.context.enter_active_request(); + let start = Instant::now(); + let co_processor_result = payload.call(http_client.clone(), &coprocessor_url).await; + let duration = start.elapsed().as_secs_f64(); + drop(guard); + tracing::info!( + histogram.apollo.router.operations.coprocessor.duration = duration, + coprocessor.stage = %PipelineStep::SupergraphResponse, + ); + + tracing::debug!(?co_processor_result, "co-processor returned"); + let co_processor_output = co_processor_result?; + + validate_coprocessor_output(&co_processor_output, PipelineStep::SupergraphResponse)?; + + // Third, process our reply and act on the contents. Our processing logic is + // that we replace "bits" of our incoming response with the updated bits if they + // are present in our co_processor_output. If they aren't present, just use the + // bits that we sent to the co_processor. + let new_body: crate::response::Response = match co_processor_output.body { + Some(value) => serde_json::from_value(value)?, + None => first, + }; + + if let Some(control) = co_processor_output.control { + parts.status = control.get_http_status()? + } + + if let Some(context) = co_processor_output.context { + for (key, value) in context.try_into_iter()? { + response + .context + .upsert_json_value(key, move |_current| value); + } + } + + if let Some(headers) = co_processor_output.headers { + parts.headers = internalize_header_map(headers)?; + } + + // Clone all the bits we need + let context = response.context.clone(); + let map_context = response.context.clone(); + + // Map the rest of our body to process subsequent chunks of response + let mapped_stream = rest + .then(move |deferred_response| { + let generator_client = http_client.clone(); + let generator_coprocessor_url = coprocessor_url.clone(); + let generator_map_context = map_context.clone(); + let generator_sdl_to_send = sdl_to_send.clone(); + + async move { + let body_to_send = response_config.body.then(|| { + serde_json::to_value(&deferred_response).expect("serialization will not fail") + }); + let context_to_send = response_config + .context + .then(|| generator_map_context.clone()); + + // Note: We deliberately DO NOT send headers or status_code even if the user has + // requested them. That's because they are meaningless on a deferred response and + // providing them will be a source of confusion. + let payload = Externalizable::router_builder() + .stage(PipelineStep::SupergraphResponse) + .and_id(TraceId::maybe_new().map(|id| id.to_string())) + .and_body(body_to_send) + .and_context(context_to_send) + .and_sdl(generator_sdl_to_send) + .build(); + + // Second, call our co-processor and get a reply. + tracing::debug!(?payload, "externalized output"); + let guard = generator_map_context.enter_active_request(); + let co_processor_result = payload + .call(generator_client, &generator_coprocessor_url) + .await; + drop(guard); + tracing::debug!(?co_processor_result, "co-processor returned"); + let co_processor_output = co_processor_result?; + + validate_coprocessor_output( + &co_processor_output, + PipelineStep::SupergraphResponse, + )?; + + // Third, process our reply and act on the contents. Our processing logic is + // that we replace "bits" of our incoming response with the updated bits if they + // are present in our co_processor_output. If they aren't present, just use the + // bits that we sent to the co_processor. + let new_deferred_response: crate::response::Response = + match co_processor_output.body { + Some(value) => serde_json::from_value(value)?, + None => deferred_response, + }; + + if let Some(context) = co_processor_output.context { + for (key, value) in context.try_into_iter()? { + generator_map_context.upsert_json_value(key, move |_current| value); + } + } + + // We return the deferred_response into our stream of response chunks + Ok(new_deferred_response) + } + }) + .map(|res: Result| match res { + Ok(response) => response, + Err(e) => { + tracing::error!("coprocessor error handling deferred supergraph response: {e}"); + response::Response::builder() + .error( + Error::builder() + .message("Internal error handling deferred response") + .extension_code("INTERNAL_ERROR") + .build(), + ) + .build() + } + }); + + // Create our response stream which consists of our first body chained with the + // rest of the responses in our mapped stream. + let stream = once(ready(new_body)).chain(mapped_stream).boxed(); + + // Finally, return a response which has a Body that wraps our stream of response chunks. + Ok(supergraph::Response { + context, + response: http::Response::from_parts(parts, stream), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use futures::future::BoxFuture; + use http::StatusCode; + use hyper::Body; + use serde_json::json; + use tower::BoxError; + use tower::ServiceExt; + + use super::super::*; + use super::*; + use crate::plugin::test::MockHttpClientService; + use crate::plugin::test::MockSupergraphService; + use crate::services::supergraph; + + #[allow(clippy::type_complexity)] + pub(crate) fn mock_with_callback( + callback: fn( + hyper::Request, + ) -> BoxFuture<'static, Result, BoxError>>, + ) -> MockHttpClientService { + let mut mock_http_client = MockHttpClientService::new(); + mock_http_client.expect_clone().returning(move || { + let mut mock_http_client = MockHttpClientService::new(); + + mock_http_client.expect_clone().returning(move || { + let mut mock_http_client = MockHttpClientService::new(); + mock_http_client.expect_call().returning(callback); + mock_http_client + }); + mock_http_client + }); + + mock_http_client + } + + #[allow(clippy::type_complexity)] + fn mock_with_deferred_callback( + callback: fn( + hyper::Request, + ) -> BoxFuture<'static, Result, BoxError>>, + ) -> MockHttpClientService { + let mut mock_http_client = MockHttpClientService::new(); + mock_http_client.expect_clone().returning(move || { + let mut mock_http_client = MockHttpClientService::new(); + mock_http_client.expect_clone().returning(move || { + let mut mock_http_client = MockHttpClientService::new(); + mock_http_client.expect_clone().returning(move || { + let mut mock_http_client = MockHttpClientService::new(); + mock_http_client.expect_call().returning(callback); + mock_http_client + }); + mock_http_client + }); + mock_http_client + }); + + mock_http_client + } + + #[tokio::test] + async fn external_plugin_supergraph_request() { + let supergraph_stage = SupergraphStage { + request: SupergraphRequestConf { + headers: false, + context: false, + body: true, + sdl: false, + method: false, + }, + response: Default::default(), + }; + + // This will never be called because we will fail at the coprocessor. + let mut mock_supergraph_service = MockSupergraphService::new(); + + mock_supergraph_service + .expect_call() + .returning(|req: supergraph::Request| { + // Let's assert that the subgraph request has been transformed as it should have. + assert_eq!( + req.supergraph_request.headers().get("cookie").unwrap(), + "tasty_cookie=strawberry" + ); + + assert_eq!( + req.context + .get::<&str, u8>("this-is-a-test-context") + .unwrap() + .unwrap(), + 42 + ); + + // The subgraph uri should have changed + assert_eq!( + Some("MyQuery"), + req.supergraph_request.body().operation_name.as_deref() + ); + + // The query should have changed + assert_eq!( + "query Long {\n me {\n name\n}\n}", + req.supergraph_request.body().query.as_ref().unwrap() + ); + + Ok(supergraph::Response::builder() + .data(json!({ "test": 1234_u32 })) + .errors(Vec::new()) + .extensions(crate::json_ext::Object::new()) + .context(req.context) + .build() + .unwrap()) + }); + + let mock_http_client = mock_with_callback(move |_: hyper::Request| { + Box::pin(async { + Ok(hyper::Response::builder() + .body(Body::from( + r#"{ + "version": 1, + "stage": "SupergraphRequest", + "control": "continue", + "headers": { + "cookie": [ + "tasty_cookie=strawberry" + ], + "content-type": [ + "application/json" + ], + "host": [ + "127.0.0.1:4000" + ], + "apollo-federation-include-trace": [ + "ftv1" + ], + "apollographql-client-name": [ + "manual" + ], + "accept": [ + "*/*" + ], + "user-agent": [ + "curl/7.79.1" + ], + "content-length": [ + "46" + ] + }, + "body": { + "query": "query Long {\n me {\n name\n}\n}", + "operationName": "MyQuery" + }, + "context": { + "entries": { + "accepts-json": false, + "accepts-wildcard": true, + "accepts-multipart": false, + "this-is-a-test-context": 42 + } + }, + "serviceName": "service name shouldn't change", + "uri": "http://thisurihaschanged" + }"#, + )) + .unwrap()) + }) + }); + + let service = supergraph_stage.as_service( + mock_http_client, + mock_supergraph_service.boxed(), + "http://test".to_string(), + Arc::new("".to_string()), + ); + + let request = supergraph::Request::fake_builder().build().unwrap(); + + assert_eq!( + serde_json_bytes::json!({ "test": 1234_u32 }), + service + .oneshot(request) + .await + .unwrap() + .response + .into_body() + .next() + .await + .unwrap() + .data + .unwrap() + ); + } + + #[tokio::test] + async fn external_plugin_supergraph_request_controlflow_break() { + let supergraph_stage = SupergraphStage { + request: SupergraphRequestConf { + headers: false, + context: false, + body: true, + sdl: false, + method: false, + }, + response: Default::default(), + }; + + // This will never be called because we will fail at the coprocessor. + let mock_supergraph_service = MockSupergraphService::new(); + + let mock_http_client = mock_with_callback(move |_: hyper::Request| { + Box::pin(async { + Ok(hyper::Response::builder() + .body(Body::from( + r#"{ + "version": 1, + "stage": "SupergraphRequest", + "control": { + "break": 200 + }, + "body": { + "errors": [{ "message": "my error message" }] + }, + "context": { + "entries": { + "testKey": true + } + }, + "headers": { + "aheader": ["a value"] + } + }"#, + )) + .unwrap()) + }) + }); + + let service = supergraph_stage.as_service( + mock_http_client, + mock_supergraph_service.boxed(), + "http://test".to_string(), + Arc::new("".to_string()), + ); + + let request = supergraph::Request::fake_builder().build().unwrap(); + + let crate::services::supergraph::Response { + mut response, + context, + } = service.oneshot(request).await.unwrap(); + + assert!(context.get::<_, bool>("testKey").unwrap().unwrap()); + + let value = response.headers().get("aheader").unwrap(); + + assert_eq!("a value", value); + + assert_eq!( + "my error message", + response.body_mut().next().await.unwrap().errors[0] + .message + .as_str() + ); + } + + #[tokio::test] + async fn external_plugin_supergraph_response() { + let supergraph_stage = SupergraphStage { + response: SupergraphResponseConf { + headers: true, + context: true, + body: true, + sdl: true, + status_code: false, + }, + request: Default::default(), + }; + + let mut mock_supergraph_service = MockSupergraphService::new(); + + mock_supergraph_service + .expect_call() + .returning(|req: supergraph::Request| { + Ok(supergraph::Response::builder() + .data(json!({ "test": 1234_u32 })) + .errors(Vec::new()) + .extensions(crate::json_ext::Object::new()) + .context(req.context) + .build() + .unwrap()) + }); + + let mock_http_client = mock_with_deferred_callback(move |res: hyper::Request| { + Box::pin(async { + let deserialized_response: Externalizable = + serde_json::from_slice(&hyper::body::to_bytes(res.into_body()).await.unwrap()) + .unwrap(); + + assert_eq!(EXTERNALIZABLE_VERSION, deserialized_response.version); + assert_eq!( + PipelineStep::SupergraphResponse.to_string(), + deserialized_response.stage + ); + + assert_eq!( + json! {{"data":{ "test": 1234_u32 }}}, + deserialized_response.body.unwrap() + ); + + let input = json!( + { + "version": 1, + "stage": "SupergraphResponse", + "control": { + "break": 400 + }, + "id": "1b19c05fdafc521016df33148ad63c1b", + "headers": { + "cookie": [ + "tasty_cookie=strawberry" + ], + "content-type": [ + "application/json" + ], + "host": [ + "127.0.0.1:4000" + ], + "apollo-federation-include-trace": [ + "ftv1" + ], + "apollographql-client-name": [ + "manual" + ], + "accept": [ + "*/*" + ], + "user-agent": [ + "curl/7.79.1" + ], + "content-length": [ + "46" + ] + }, + "body": { + "data": { "test": 42 } + }, + "context": { + "entries": { + "accepts-json": false, + "accepts-wildcard": true, + "accepts-multipart": false, + "this-is-a-test-context": 42 + } + }, + "sdl": "the sdl shouldnt change" + }); + Ok(hyper::Response::builder() + .body(Body::from(serde_json::to_string(&input).unwrap())) + .unwrap()) + }) + }); + + let service = supergraph_stage.as_service( + mock_http_client, + mock_supergraph_service.boxed(), + "http://test".to_string(), + Arc::new("".to_string()), + ); + + let request = supergraph::Request::canned_builder().build().unwrap(); + + let mut res = service.oneshot(request).await.unwrap(); + + // Let's assert that the router request has been transformed as it should have. + assert_eq!(res.response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + res.response.headers().get("cookie").unwrap(), + "tasty_cookie=strawberry" + ); + + assert_eq!( + res.context + .get::<&str, u8>("this-is-a-test-context") + .unwrap() + .unwrap(), + 42 + ); + + let body = res.response.body_mut().next().await.unwrap(); + // the body should have changed: + assert_eq!( + json!({ "data": { "test": 42_u32 } }), + serde_json::to_value(&body).unwrap(), + ); + } +} diff --git a/apollo-router/src/plugins/coprocessor_test.rs b/apollo-router/src/plugins/coprocessor/test.rs similarity index 99% rename from apollo-router/src/plugins/coprocessor_test.rs rename to apollo-router/src/plugins/coprocessor/test.rs index 76d4177768..3f4b50876e 100644 --- a/apollo-router/src/plugins/coprocessor_test.rs +++ b/apollo-router/src/plugins/coprocessor/test.rs @@ -17,7 +17,7 @@ mod tests { use tower::BoxError; use tower::ServiceExt; - use super::super::coprocessor::*; + use super::super::*; use crate::plugin::test::MockHttpClientService; use crate::plugin::test::MockRouterService; use crate::plugin::test::MockSubgraphService; diff --git a/apollo-router/src/plugins/mod.rs b/apollo-router/src/plugins/mod.rs index df3252d3df..14526d158f 100644 --- a/apollo-router/src/plugins/mod.rs +++ b/apollo-router/src/plugins/mod.rs @@ -23,8 +23,6 @@ macro_rules! schemar_fn { pub(crate) mod authentication; pub(crate) mod authorization; mod coprocessor; -#[cfg(test)] -mod coprocessor_test; pub(crate) mod csrf; mod expose_query_plan; mod forbid_mutations; diff --git a/apollo-router/src/services/external.rs b/apollo-router/src/services/external.rs index 440f399d1d..2161ac053d 100644 --- a/apollo-router/src/services/external.rs +++ b/apollo-router/src/services/external.rs @@ -131,6 +131,42 @@ where } } + #[builder(visibility = "pub(crate)")] + /// This is the constructor (or builder) to use when constructing a Supergraph + /// `Externalizable`. + /// + fn supergraph_new( + stage: PipelineStep, + control: Option, + id: Option, + headers: Option>>, + body: Option, + context: Option, + status_code: Option, + method: Option, + sdl: Option, + ) -> Self { + assert!(matches!( + stage, + PipelineStep::SupergraphRequest | PipelineStep::SupergraphResponse + )); + Externalizable { + version: EXTERNALIZABLE_VERSION, + stage: stage.to_string(), + control, + id, + headers, + body, + context, + status_code, + sdl, + uri: None, + path: None, + method, + service_name: None, + } + } + #[builder(visibility = "pub(crate)")] /// This is the constructor (or builder) to use when constructing a Subgraph /// `Externalizable`. diff --git a/docs/source/customizations/coprocessor.mdx b/docs/source/customizations/coprocessor.mdx index b01cf4e301..8fd14fe81f 100644 --- a/docs/source/customizations/coprocessor.mdx +++ b/docs/source/customizations/coprocessor.mdx @@ -48,7 +48,7 @@ flowchart TB; > This diagram shows request execution proceeding "down" from a client, through the router, to individual subgraphs. Execution then proceeds back "up" to the client in the reverse order. -As shown in the diagram above, only the `RouterService` and `SubgraphService` of the [request-handling lifecycle](./rhai/#router-request-lifecycle) can send these POST requests (also called **coprocessor requests**). +As shown in the diagram above, the `RouterService`, `SupergraphService` and `SubgraphService` steps of the [request-handling lifecycle](./rhai/#router-request-lifecycle) can send these POST requests (also called **coprocessor requests**). Each supported service can send its coprocessor requests at two different **stages**: @@ -92,6 +92,19 @@ coprocessor: context: false sdl: false status_code: false + supergraph: # This coprocessor hooks into the `SupergraphService` + request: # By including this key, the `SupergraphService` sends a coprocessor request whenever it first receives a client request. + headers: true # These boolean properties indicate which request data to include in the coprocessor request. All are optional and false by default. + body: false + context: false + sdl: false + method: false + response: # By including this key, the `SupergraphService` sends a coprocessor request whenever it's about to send response data to a client (including incremental data via @defer). + headers: true + body: false + context: false + sdl: false + status_code: false subgraph: all: request: # By including this key, the `SubgraphService` sends a coprocessor request whenever it is about to make a request to a subgraph. From cfc5e8872966d6cb044ba2f4a0b03797580a4ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9e?= Date: Thu, 14 Sep 2023 10:29:09 +0200 Subject: [PATCH 15/51] Fix error response on large numbers in query transformations (#3820) In GraphQL source text, you can write an integer outside the int32 range if a float is expected. apollo-rs parses these numbers as f64 inside `hir::Value::Int` because at that time, the expected type of the value is not known, especially because it's possible to parse queries without knowledge of the schema. In the defer label transformation, any apollo-rs `hir::Value::Int` was treated as an i32. This would cause an error if it's one such large integer. The fully proper way to convert `hir::Value` to another type (ie. encoder or serde_json, as is done elsewhere) would be to match on the actual GraphQL type declared by the schema. But the simpler thing to do is to just use a Float if the value is too big for an Int. This will round-trip correctly. See test for an example that used to fail: https://github.com/apollographql/router/blob/5b0f608bd4ff613bdcc1a99e0c1a13f7735f0070/apollo-router/src/query_planner/labeler.rs#L128-L129 --- **Checklist** Complete the checklist (and note appropriate exceptions) before the PR is marked ready-for-review. - [x] Changes are compatible[^1] - [ ] Documentation[^2] completed - [ ] Performance impact assessed and acceptable - Tests added and passing[^3] - [x] Unit Tests - [ ] Integration Tests - [ ] Manual Tests **Exceptions** *Note any exceptions here* **Notes** [^1]: It may be appropriate to bring upcoming changes to the attention of other (impacted) groups. Please endeavour to do this before seeking PR approval. The mechanism for doing this will vary considerably, so use your judgement as to how and when to do this. [^2]: Configuration is an important part of many changes. Where applicable please try to document configuration examples. [^3]: Tick whichever testing boxes are applicable. If you are adding Manual Tests, please document the manual testing (extensively) in the Exceptions. --- .changesets/fix_renee_fix_large_ints.md | 22 +++++++++++++++++++ apollo-router/src/query_planner/labeler.rs | 16 ++++++++++++++ ...er__tests__large_float_written_as_int.snap | 8 +++++++ apollo-router/src/spec/query/transform.rs | 7 +++--- 4 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 .changesets/fix_renee_fix_large_ints.md create mode 100644 apollo-router/src/query_planner/snapshots/apollo_router__query_planner__labeler__tests__large_float_written_as_int.snap diff --git a/.changesets/fix_renee_fix_large_ints.md b/.changesets/fix_renee_fix_large_ints.md new file mode 100644 index 0000000000..756249f92c --- /dev/null +++ b/.changesets/fix_renee_fix_large_ints.md @@ -0,0 +1,22 @@ +### Fix error response on large numbers in query transformations ([PR #3820](https://github.com/apollographql/router/pull/3820)) + +This bug caused the router to reject operations where a large hardcoded integer was used as input for a Float field: + +```graphql +# Schema +type Query { + field(argument: Float): Int! +} +# Operation +{ + field(argument: 123456789123) +} +``` + +Now the number is correctly interpreted as a Float. +This bug only affected hardcoded numbers, not numbers provided through variables. + + +--- + +By [@goto-bus-stop](https://github.com/goto-bus-stop) in https://github.com/apollographql/router/pull/3820 \ No newline at end of file diff --git a/apollo-router/src/query_planner/labeler.rs b/apollo-router/src/query_planner/labeler.rs index 7bbb095eb7..45eac5215c 100644 --- a/apollo-router/src/query_planner/labeler.rs +++ b/apollo-router/src/query_planner/labeler.rs @@ -116,3 +116,19 @@ pub(crate) fn directive( Ok(encoder_directive) } + +#[cfg(test)] +mod tests { + use apollo_compiler::ApolloCompiler; + + use super::add_defer_labels; + + #[test] + fn large_float_written_as_int() { + let mut compiler = ApolloCompiler::new(); + compiler.add_type_system("type Query { field(id: Float): String! }", "schema.graphql"); + let file_id = compiler.add_executable(r#"{ field(id: 1234567890123) }"#, "query.graphql"); + let result = add_defer_labels(file_id, &compiler).unwrap(); + insta::assert_snapshot!(result); + } +} diff --git a/apollo-router/src/query_planner/snapshots/apollo_router__query_planner__labeler__tests__large_float_written_as_int.snap b/apollo-router/src/query_planner/snapshots/apollo_router__query_planner__labeler__tests__large_float_written_as_int.snap new file mode 100644 index 0000000000..8be08a97cc --- /dev/null +++ b/apollo-router/src/query_planner/snapshots/apollo_router__query_planner__labeler__tests__large_float_written_as_int.snap @@ -0,0 +1,8 @@ +--- +source: apollo-router/src/query_planner/labeler.rs +expression: result +--- +query { + field(id: 1234567890123) +} + diff --git a/apollo-router/src/spec/query/transform.rs b/apollo-router/src/spec/query/transform.rs index 76bfe0f9be..2a5571fc37 100644 --- a/apollo-router/src/spec/query/transform.rs +++ b/apollo-router/src/spec/query/transform.rs @@ -369,9 +369,10 @@ pub(crate) fn ty(hir: &hir::Type) -> apollo_encoder::Type_ { pub(crate) fn value(hir: &hir::Value) -> Result { Ok(match hir { hir::Value::Variable(val) => apollo_encoder::Value::Variable(val.name().into()), - hir::Value::Int { value, .. } => { - apollo_encoder::Value::Int(value.to_i32_checked().ok_or("Int value overflows i32")?) - } + hir::Value::Int { value, .. } => value + .to_i32_checked() + .map(apollo_encoder::Value::Int) + .unwrap_or_else(|| apollo_encoder::Value::Float(value.get())), hir::Value::Float { value, .. } => apollo_encoder::Value::Float(value.get()), hir::Value::String { value, .. } => apollo_encoder::Value::String(value.clone()), hir::Value::Boolean { value, .. } => apollo_encoder::Value::Boolean(*value), From 9b618d0c7f7221b7a11b67532c60edf4d3b2c568 Mon Sep 17 00:00:00 2001 From: Gary Pennington Date: Thu, 14 Sep 2023 10:00:20 +0100 Subject: [PATCH 16/51] fix the diy build_docker_image.sh script (#3824) Required to build deno. Fixes #3823 --- **Checklist** Complete the checklist (and note appropriate exceptions) before the PR is marked ready-for-review. - [x] Changes are compatible[^1] **Exceptions** *Note any exceptions here* **Notes** [^1]: It may be appropriate to bring upcoming changes to the attention of other (impacted) groups. Please endeavour to do this before seeking PR approval. The mechanism for doing this will vary considerably, so use your judgement as to how and when to do this. [^2]: Configuration is an important part of many changes. Where applicable please try to document configuration examples. [^3]: Tick whichever testing boxes are applicable. If you are adding Manual Tests, please document the manual testing (extensively) in the Exceptions. --- .changesets/fix_garypen_3823_fix_diy.md | 7 +++++++ dockerfiles/diy/dockerfiles/Dockerfile.repo | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .changesets/fix_garypen_3823_fix_diy.md diff --git a/.changesets/fix_garypen_3823_fix_diy.md b/.changesets/fix_garypen_3823_fix_diy.md new file mode 100644 index 0000000000..3da4807001 --- /dev/null +++ b/.changesets/fix_garypen_3823_fix_diy.md @@ -0,0 +1,7 @@ +### fix the diy build_docker_image.sh script ([Issue #3823](https://github.com/apollographql/router/issues/3823)) + +The diy `build_docker_image.sh` script was broken by the latest `deno` update due to a new requirement to have `cmake` when compiling the router. + +This adds `cmake` to the build step for diy builds to fix the problem. + +By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3824 diff --git a/dockerfiles/diy/dockerfiles/Dockerfile.repo b/dockerfiles/diy/dockerfiles/Dockerfile.repo index 99f03349e5..ebdac87d39 100644 --- a/dockerfiles/diy/dockerfiles/Dockerfile.repo +++ b/dockerfiles/diy/dockerfiles/Dockerfile.repo @@ -10,7 +10,8 @@ RUN apt-get update RUN apt-get -y install \ npm \ nodejs \ - protobuf-compiler + protobuf-compiler \ + cmake # Add rustfmt since build requires it RUN rustup component add rustfmt From 641bfac160d58439719d44d11482ccdc7fd8567b Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Fri, 15 Sep 2023 14:16:29 +0200 Subject: [PATCH 17/51] fix(telemetry): do not display otel bug if the trace is sampled (#3832) We changed the way we are sampling spans. If you had logs in an evicted span it displays that log `Unable to find OtelData in extensions; this is a bug` which is incorrect, it's not a bug, we just can't display the `trace_id` because there is no `trace_id` as the span has not been sampled. --------- Signed-off-by: Benjamin Coenen <5719034+bnjjj@users.noreply.github.com> --- .../fix_bnjjj_fix_otel_data_from_log.md | 5 +++++ .../src/plugins/telemetry/formatters/text.rs | 8 ++++++- apollo-router/src/plugins/telemetry/reload.rs | 22 +++++++++++++++---- 3 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 .changesets/fix_bnjjj_fix_otel_data_from_log.md diff --git a/.changesets/fix_bnjjj_fix_otel_data_from_log.md b/.changesets/fix_bnjjj_fix_otel_data_from_log.md new file mode 100644 index 0000000000..8d2fb003cc --- /dev/null +++ b/.changesets/fix_bnjjj_fix_otel_data_from_log.md @@ -0,0 +1,5 @@ +### fix(telemetry): do not display otel bug if the trace is sampled ([PR #3832](https://github.com/apollographql/router/pull/3832)) + +We changed the way we are sampling spans. If you had logs in an evicted span it displays that log `Unable to find OtelData in extensions; this is a bug` which is incorrect, it's not a bug, we just can't display the `trace_id` because there is no `trace_id` as the span has not been sampled. + +By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3832 \ No newline at end of file diff --git a/apollo-router/src/plugins/telemetry/formatters/text.rs b/apollo-router/src/plugins/telemetry/formatters/text.rs index 1ed1a79b2c..1ea09dcc31 100644 --- a/apollo-router/src/plugins/telemetry/formatters/text.rs +++ b/apollo-router/src/plugins/telemetry/formatters/text.rs @@ -17,6 +17,8 @@ use tracing_subscriber::fmt::time::SystemTime; use tracing_subscriber::fmt::FmtContext; use tracing_subscriber::registry::LookupSpan; +use crate::plugins::telemetry::reload::IsSampled; + #[derive(Debug, Clone)] pub(crate) struct TextFormatter { pub(crate) timer: SystemTime, @@ -185,7 +187,11 @@ impl TextFormatter { } writer.write_char(' ')?; } - None => eprintln!("Unable to find OtelData in extensions; this is a bug"), + None => { + if span.is_sampled() { + eprintln!("Unable to find OtelData in extensions; this is a bug"); + } + } } } diff --git a/apollo-router/src/plugins/telemetry/reload.rs b/apollo-router/src/plugins/telemetry/reload.rs index 96296341e5..2dcf270f41 100644 --- a/apollo-router/src/plugins/telemetry/reload.rs +++ b/apollo-router/src/plugins/telemetry/reload.rs @@ -21,6 +21,7 @@ use tracing_subscriber::layer::Layer; use tracing_subscriber::layer::Layered; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::registry::LookupSpan; +use tracing_subscriber::registry::SpanRef; use tracing_subscriber::reload::Handle; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; @@ -220,10 +221,7 @@ where .id() .and_then(|id| cx.span(id)) { - // if this extension is set, that means the parent span was accepted, and so the - // entire trace is accepted - let extensions = spanref.extensions(); - return extensions.get::().is_some(); + return spanref.is_sampled(); } // we only make the sampling decision on the root span. If we reach here for any other span, @@ -257,6 +255,22 @@ where } struct SampledSpan; + +pub(crate) trait IsSampled { + fn is_sampled(&self) -> bool; +} + +impl<'a, T> IsSampled for SpanRef<'a, T> +where + T: tracing_subscriber::registry::LookupSpan<'a>, +{ + fn is_sampled(&self) -> bool { + // if this extension is set, that means the parent span was accepted, and so the + // entire trace is accepted + let extensions = self.extensions(); + extensions.get::().is_some() + } +} /// prevents span fields from being formatted to a string when writing logs pub(crate) struct NullFieldFormatter; From 0bf44407908787b1c13f92678f6c408491834b35 Mon Sep 17 00:00:00 2001 From: Gary Pennington Date: Fri, 15 Sep 2023 14:16:45 +0100 Subject: [PATCH 18/51] Check if docker images and helm charts exist before creating them (#3825) This improves the resilience of our pipeline. fixes: #3663 --- **Checklist** Complete the checklist (and note appropriate exceptions) before the PR is marked ready-for-review. - [x] Changes are compatible[^1] **Exceptions** *Note any exceptions here* **Notes** [^1]: It may be appropriate to bring upcoming changes to the attention of other (impacted) groups. Please endeavour to do this before seeking PR approval. The mechanism for doing this will vary considerably, so use your judgement as to how and when to do this. [^2]: Configuration is an important part of many changes. Where applicable please try to document configuration examples. [^3]: Tick whichever testing boxes are applicable. If you are adding Manual Tests, please document the manual testing (extensively) in the Exceptions. --- .changesets/maint_garypen_3663_no_overwrites.md | 5 +++++ .circleci/config.yml | 8 ++++++++ 2 files changed, 13 insertions(+) create mode 100644 .changesets/maint_garypen_3663_no_overwrites.md diff --git a/.changesets/maint_garypen_3663_no_overwrites.md b/.changesets/maint_garypen_3663_no_overwrites.md new file mode 100644 index 0000000000..1a155f7353 --- /dev/null +++ b/.changesets/maint_garypen_3663_no_overwrites.md @@ -0,0 +1,5 @@ +### Check if docker images and helm charts exist before creating them ([Issue #3663](https://github.com/apollographql/router/issues/3663)) + +This improves the resilience of our pipeline. + +By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3825 \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml index 840e33a254..63decfa09b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -761,6 +761,10 @@ jobs: docker buildx inspect --bootstrap # Note: GH Token owned by apollo-bot2, no expire echo ${GITHUB_OCI_TOKEN} | docker login ghcr.io -u apollo-bot2 --password-stdin + # To prevent overwrite, check to see if our images already exists + # If the manifest command succeeds, the images already exist + docker manifest inspect ${ROUTER_TAG}:${VERSION} > /dev/null && exit 1 + docker manifest inspect ${ROUTER_TAG}:${VERSION}-debug > /dev/null && exit 1 # Build and push debug image docker buildx build --platform linux/amd64,linux/arm64 --push --build-arg DEBUG_IMAGE="true" --build-arg ROUTER_RELEASE=${VERSION} -f dockerfiles/Dockerfile.router -t ${ROUTER_TAG}:${VERSION}-debug . # Build and push release image @@ -776,6 +780,10 @@ jobs: CHART=$(ls -t router*.tgz| head -1) # Note: GH Token owned by apollo-bot2, no expire echo ${GITHUB_OCI_TOKEN} | helm registry login -u apollo-bot2 --password-stdin ghcr.io + # To prevent overwrite, check to see if our chart already exists + # If the show all command succeeds, the chart already exists + VERSION=$(basename ${CHART} .tgz) + helm show all oci://ghcr.io/apollographql/helm-charts/router --version ${VERSION} > /dev/null && exit 1 # Push chart to repository helm push ${CHART} oci://ghcr.io/apollographql/helm-charts From f831f68afb7eb06b67f87d1aedb1839e82587b6a Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Fri, 15 Sep 2023 15:59:07 +0200 Subject: [PATCH 19/51] feat(telemetry): add metrics for query plan warmup and schema loading (#3807) It adds histogram metrics for `apollo_router_query_planning_warmup_duration` and `apollo_router_schema_load_duration`. Example in Prometheus: ``` # HELP apollo_router_query_planning_warmup_duration apollo_router_query_planning_warmup_duration # TYPE apollo_router_query_planning_warmup_duration histogram apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.5"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="2.5"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="5"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="10"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="20"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="100"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1000"} 1 apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="+Inf"} 1 apollo_router_query_planning_warmup_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.022390619 apollo_router_query_planning_warmup_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 1 # HELP apollo_router_schema_load_duration apollo_router_schema_load_duration # TYPE apollo_router_schema_load_duration histogram apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.5"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="2.5"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="5"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="10"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="20"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="100"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1000"} 8 apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="+Inf"} 8 apollo_router_schema_load_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.023486205999999996 apollo_router_schema_load_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 8 ``` Fixes #3767 --------- Signed-off-by: Benjamin Coenen <5719034+bnjjj@users.noreply.github.com> --- .changesets/feat_bnjjj_fix_3767.md | 42 +++++++++++++++++++ .../src/plugins/telemetry/metrics/filter.rs | 6 ++- apollo-router/src/plugins/telemetry/utils.rs | 38 +++++++++++++++++ .../src/query_planner/bridge_query_planner.rs | 2 +- .../query_planner/caching_query_planner.rs | 6 +++ apollo-router/src/spec/schema.rs | 5 +++ docs/source/configuration/metrics.mdx | 3 ++ 7 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 .changesets/feat_bnjjj_fix_3767.md diff --git a/.changesets/feat_bnjjj_fix_3767.md b/.changesets/feat_bnjjj_fix_3767.md new file mode 100644 index 0000000000..a79b1e059d --- /dev/null +++ b/.changesets/feat_bnjjj_fix_3767.md @@ -0,0 +1,42 @@ +### feat(telemetry): add metrics for query plan warmup and schema load ([Issue #3767](https://github.com/apollographql/router/issues/3767)) + +It adds histogram metrics for `apollo_router_query_planning_warmup_duration` and `apollo_router_schema_load_duration`. + +Example in Prometheus: + +``` +# HELP apollo_router_query_planning_warmup_duration apollo_router_query_planning_warmup_duration +# TYPE apollo_router_query_planning_warmup_duration histogram +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.5"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="2.5"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="5"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="10"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="20"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="100"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1000"} 1 +apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="+Inf"} 1 +apollo_router_query_planning_warmup_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.022390619 +apollo_router_query_planning_warmup_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 1 +# HELP apollo_router_schema_load_duration apollo_router_schema_load_duration +# TYPE apollo_router_schema_load_duration histogram +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.5"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="2.5"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="5"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="10"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="20"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="100"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1000"} 8 +apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="+Inf"} 8 +apollo_router_schema_load_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.023486205999999996 +apollo_router_schema_load_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 8 +``` + +By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3807 diff --git a/apollo-router/src/plugins/telemetry/metrics/filter.rs b/apollo-router/src/plugins/telemetry/metrics/filter.rs index 919a073cd7..929a8a9d17 100644 --- a/apollo-router/src/plugins/telemetry/metrics/filter.rs +++ b/apollo-router/src/plugins/telemetry/metrics/filter.rs @@ -37,8 +37,10 @@ impl FilterMeterProvider { FilterMeterProvider::builder() .delegate(delegate) .allow( - Regex::new(r"apollo\.(graphos\.cloud|router\.(operations?|config))(\..*|$)") - .expect("regex should have been valid"), + Regex::new( + r"apollo\.(graphos\.cloud|router\.(operations?|config|schema|query))(\..*|$)", + ) + .expect("regex should have been valid"), ) .build() } diff --git a/apollo-router/src/plugins/telemetry/utils.rs b/apollo-router/src/plugins/telemetry/utils.rs index f845192c2d..42b57f122c 100644 --- a/apollo-router/src/plugins/telemetry/utils.rs +++ b/apollo-router/src/plugins/telemetry/utils.rs @@ -1,3 +1,6 @@ +use std::time::Duration; +use std::time::Instant; + use tracing_core::field::Value; pub(crate) trait TracingUtils { @@ -13,3 +16,38 @@ impl TracingUtils for bool { } } } + +/// Timer implementing Drop to automatically compute the duration between the moment it has been created until it's dropped +///```ignore +/// Timer::new(|duration| { +/// tracing::info!(histogram.apollo_router_test = duration.as_secs_f64()); +/// }) +/// ``` +pub(crate) struct Timer +where + F: FnOnce(Duration), +{ + start: Instant, + f: Option, +} + +impl Timer +where + F: FnOnce(Duration), +{ + pub(crate) fn new(f: F) -> Self { + Self { + start: Instant::now(), + f: f.into(), + } + } +} + +impl Drop for Timer +where + F: FnOnce(Duration), +{ + fn drop(&mut self) { + self.f.take().expect("f must exist")(self.start.elapsed()) + } +} diff --git a/apollo-router/src/query_planner/bridge_query_planner.rs b/apollo-router/src/query_planner/bridge_query_planner.rs index 4a677aaad1..a1008498c0 100644 --- a/apollo-router/src/query_planner/bridge_query_planner.rs +++ b/apollo-router/src/query_planner/bridge_query_planner.rs @@ -476,7 +476,7 @@ impl Service for BridgeQueryPlanner { ) .await; let duration = start.elapsed().as_secs_f64(); - tracing::info!(histogram.apollo_router_query_planning_time = duration,); + tracing::info!(histogram.apollo_router_query_planning_time = duration); match res { Ok(query_planner_content) => Ok(QueryPlannerResponse::builder() diff --git a/apollo-router/src/query_planner/caching_query_planner.rs b/apollo-router/src/query_planner/caching_query_planner.rs index 034f428f32..52876b6e0e 100644 --- a/apollo-router/src/query_planner/caching_query_planner.rs +++ b/apollo-router/src/query_planner/caching_query_planner.rs @@ -22,6 +22,7 @@ use crate::error::CacheResolverError; use crate::error::QueryPlannerError; use crate::plugins::authorization::AuthorizationPlugin; use crate::plugins::authorization::CacheKeyMetadata; +use crate::plugins::telemetry::utils::Timer; use crate::query_planner::labeler::add_defer_labels; use crate::query_planner::BridgeQueryPlanner; use crate::query_planner::QueryPlanResult; @@ -107,6 +108,11 @@ where query_analysis: &QueryAnalysisLayer, cache_keys: Vec, ) { + let _timer = Timer::new(|duration| { + ::tracing::info!( + histogram.apollo.router.query.planning.warmup.duration = duration.as_secs_f64() + ); + }); let schema_id = self.schema.schema_id.clone(); let mut service = ServiceBuilder::new().service( diff --git a/apollo-router/src/spec/schema.rs b/apollo-router/src/spec/schema.rs index 744e3d2cee..1703484b9d 100644 --- a/apollo-router/src/spec/schema.rs +++ b/apollo-router/src/spec/schema.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; +use std::time::Instant; use apollo_compiler::diagnostics::ApolloDiagnostic; use apollo_compiler::ApolloCompiler; @@ -61,6 +62,7 @@ impl Schema { } pub(crate) fn parse(sdl: &str, configuration: &Configuration) -> Result { + let start = Instant::now(); let mut compiler = ApolloCompiler::new(); let id = compiler.add_type_system(sdl, "schema.graphql"); @@ -129,6 +131,9 @@ impl Schema { let mut hasher = Sha256::new(); hasher.update(sdl.as_bytes()); let schema_id = Some(format!("{:x}", hasher.finalize())); + tracing::info!( + histogram.apollo.router.schema.load.duration = start.elapsed().as_secs_f64() + ); Ok(Schema { raw_sdl: Arc::new(sdl.to_string()), diff --git a/docs/source/configuration/metrics.mdx b/docs/source/configuration/metrics.mdx index a250824e5b..f85ff5d428 100644 --- a/docs/source/configuration/metrics.mdx +++ b/docs/source/configuration/metrics.mdx @@ -103,6 +103,8 @@ The coprocessor operations metric has the following attributes: - `apollo_router_processing_time` - Time spent processing a request (outside of waiting for external or subgraph requests) in seconds. - `apollo_router_query_planning_time` - Time spent planning queries in seconds. +- `apollo_router_query_planning_warmup_duration` - Time spent planning queries in seconds. +- `apollo_router_schema_load_duration` - Time spent loading the schema in seconds. #### Uplink @@ -121,6 +123,7 @@ The coprocessor operations metric has the following attributes: Note that the initial call to uplink during router startup will not be reflected in metrics. #### Subscription + - `apollo_router_opened_subscriptions` - Number of different opened subscriptions (not the number of clients with an opened subscriptions in case it's deduplicated) - `apollo_router_deduplicated_subscriptions_total` - Number of subscriptions that has been deduplicated - `apollo_router_skipped_event_count` - Number of subscription events that has been skipped because too many events have been received from the subgraph but not yet sent to the client. From 64ffae615027e3586e7dbbae68a714035a2e3939 Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Mon, 18 Sep 2023 11:54:12 +0200 Subject: [PATCH 20/51] feat(redis): add configuration to set the timeout (#3817) It adds a configuration to set another timeout than the default one (2ms) for redis requests. It also change the default timeout to 2ms (previously set to 1ms) Example for APQ: ```yaml apq: router: cache: redis: urls: ["redis://..."] timeout: 5ms ``` Fixes #3621 --- **Checklist** Complete the checklist (and note appropriate exceptions) before the PR is marked ready-for-review. - [x] Changes are compatible[^1] - [ ] Documentation[^2] completed - [ ] Performance impact assessed and acceptable - Tests added and passing[^3] - [ ] Unit Tests - [ ] Integration Tests - [x] Manual Tests **Exceptions** *Note any exceptions here* **Notes** [^1]: It may be appropriate to bring upcoming changes to the attention of other (impacted) groups. Please endeavour to do this before seeking PR approval. The mechanism for doing this will vary considerably, so use your judgement as to how and when to do this. [^2]: Configuration is an important part of many changes. Where applicable please try to document configuration examples. [^3]: Tick whichever testing boxes are applicable. If you are adding Manual Tests, please document the manual testing (extensively) in the Exceptions. --------- Signed-off-by: Benjamin Coenen <5719034+bnjjj@users.noreply.github.com> --- .changesets/feat_bnjjj_fix_3621.md | 15 +++++++++++++++ apollo-router/src/cache/mod.rs | 14 ++++++++++---- apollo-router/src/cache/redis.rs | 8 ++++++-- apollo-router/src/cache/storage.rs | 8 +++++--- apollo-router/src/configuration/mod.rs | 6 +++++- ...onfiguration__tests__schema_generation.snap | 18 ++++++++++++++++++ apollo-router/src/introspection.rs | 2 +- .../src/plugins/traffic_shaping/mod.rs | 12 +++++++++++- .../configuration/distributed-caching.mdx | 2 ++ 9 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 .changesets/feat_bnjjj_fix_3621.md diff --git a/.changesets/feat_bnjjj_fix_3621.md b/.changesets/feat_bnjjj_fix_3621.md new file mode 100644 index 0000000000..c09c2619a5 --- /dev/null +++ b/.changesets/feat_bnjjj_fix_3621.md @@ -0,0 +1,15 @@ +### feat(redis): add configuration to set the timeout ([Issue #3621](https://github.com/apollographql/router/issues/3621)) + +It adds a configuration to set another timeout than the default one (2ms) for redis requests. It also change the default timeout to 2ms (previously set to 1ms) + +Example for APQ: +```yaml +apq: + router: + cache: + redis: + urls: ["redis://..."] + timeout: 5ms +``` + +By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3817 \ No newline at end of file diff --git a/apollo-router/src/cache/mod.rs b/apollo-router/src/cache/mod.rs index cdea508257..a577e847a7 100644 --- a/apollo-router/src/cache/mod.rs +++ b/apollo-router/src/cache/mod.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::num::NonZeroUsize; use std::sync::Arc; +use std::time::Duration; use tokio::sync::broadcast; use tokio::sync::oneshot; @@ -34,11 +35,12 @@ where pub(crate) async fn with_capacity( capacity: NonZeroUsize, redis_urls: Option>, + timeout: Option, caller: &str, ) -> Self { Self { wait_map: Arc::new(Mutex::new(HashMap::new())), - storage: CacheStorage::new(capacity, redis_urls, caller).await, + storage: CacheStorage::new(capacity, redis_urls, timeout, caller).await, } } @@ -49,6 +51,7 @@ where Self::with_capacity( config.in_memory.limit, config.redis.as_ref().map(|c| c.urls.clone()), + config.redis.as_ref().and_then(|r| r.timeout), caller, ) .await @@ -211,7 +214,8 @@ mod tests { async fn example_cache_usage() { let k = "key".to_string(); let cache = - DeduplicatingCache::with_capacity(NonZeroUsize::new(1).unwrap(), None, "test").await; + DeduplicatingCache::with_capacity(NonZeroUsize::new(1).unwrap(), None, None, "test") + .await; let entry = cache.get(&k).await; @@ -228,7 +232,8 @@ mod tests { #[test(tokio::test)] async fn it_should_enforce_cache_limits() { let cache: DeduplicatingCache = - DeduplicatingCache::with_capacity(NonZeroUsize::new(13).unwrap(), None, "test").await; + DeduplicatingCache::with_capacity(NonZeroUsize::new(13).unwrap(), None, None, "test") + .await; for i in 0..14 { let entry = cache.get(&i).await; @@ -251,7 +256,8 @@ mod tests { mock.expect_retrieve().times(1).return_const(1usize); let cache: DeduplicatingCache = - DeduplicatingCache::with_capacity(NonZeroUsize::new(10).unwrap(), None, "test").await; + DeduplicatingCache::with_capacity(NonZeroUsize::new(10).unwrap(), None, None, "test") + .await; // Let's trigger 100 concurrent gets of the same value and ensure only // one delegated retrieve is made diff --git a/apollo-router/src/cache/redis.rs b/apollo-router/src/cache/redis.rs index d414ba35f7..81cc335310 100644 --- a/apollo-router/src/cache/redis.rs +++ b/apollo-router/src/cache/redis.rs @@ -115,14 +115,18 @@ where } impl RedisCacheStorage { - pub(crate) async fn new(urls: Vec, ttl: Option) -> Result { + pub(crate) async fn new( + urls: Vec, + ttl: Option, + timeout: Option, + ) -> Result { let url = Self::preprocess_urls(urls)?; let config = RedisConfig::from_url(url.as_str())?; let client = RedisClient::new( config, Some(PerformanceConfig { - default_command_timeout_ms: 1, + default_command_timeout_ms: timeout.map(|t| t.as_millis() as u64).unwrap_or(2), ..Default::default() }), Some(ReconnectPolicy::new_exponential(0, 1, 2000, 5)), diff --git a/apollo-router/src/cache/storage.rs b/apollo-router/src/cache/storage.rs index 9996f1d23a..8106b5a864 100644 --- a/apollo-router/src/cache/storage.rs +++ b/apollo-router/src/cache/storage.rs @@ -3,6 +3,7 @@ use std::fmt::{self}; use std::hash::Hash; use std::num::NonZeroUsize; use std::sync::Arc; +use std::time::Duration; use lru::LruCache; use serde::de::DeserializeOwned; @@ -57,14 +58,15 @@ where { pub(crate) async fn new( max_capacity: NonZeroUsize, - _redis_urls: Option>, + redis_urls: Option>, + timeout: Option, caller: &str, ) -> Self { Self { caller: caller.to_string(), inner: Arc::new(Mutex::new(LruCache::new(max_capacity))), - redis: if let Some(urls) = _redis_urls { - match RedisCacheStorage::new(urls, None).await { + redis: if let Some(urls) = redis_urls { + match RedisCacheStorage::new(urls, None, timeout).await { Err(e) => { tracing::error!( "could not open connection to Redis for {} caching: {:?}", diff --git a/apollo-router/src/configuration/mod.rs b/apollo-router/src/configuration/mod.rs index 7da7e2a9ab..4126a040ee 100644 --- a/apollo-router/src/configuration/mod.rs +++ b/apollo-router/src/configuration/mod.rs @@ -20,7 +20,6 @@ use std::net::SocketAddr; use std::num::NonZeroUsize; use std::str::FromStr; use std::sync::Arc; -#[cfg(not(test))] use std::time::Duration; use derivative::Derivative; @@ -866,6 +865,11 @@ impl Default for InMemoryCache { pub(crate) struct RedisCache { /// List of URLs to the Redis cluster pub(crate) urls: Vec, + + #[serde(deserialize_with = "humantime_serde::deserialize", default)] + #[schemars(with = "Option", default)] + /// Redis request timeout (default: 2ms) + pub(crate) timeout: Option, } /// TLS related configuration options. diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index a175614292..541dbd3a7a 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -83,6 +83,12 @@ expression: "&schema" "urls" ], "properties": { + "timeout": { + "description": "Redis request timeout (default: 2ms)", + "default": null, + "type": "string", + "nullable": true + }, "urls": { "description": "List of URLs to the Redis cluster", "type": "array", @@ -1991,6 +1997,12 @@ expression: "&schema" "urls" ], "properties": { + "timeout": { + "description": "Redis request timeout (default: 2ms)", + "default": null, + "type": "string", + "nullable": true + }, "urls": { "description": "List of URLs to the Redis cluster", "type": "array", @@ -5592,6 +5604,12 @@ expression: "&schema" "urls" ], "properties": { + "timeout": { + "description": "Redis request timeout (default: 2ms)", + "default": null, + "type": "string", + "nullable": true + }, "urls": { "description": "List of URLs to the Redis cluster", "type": "array", diff --git a/apollo-router/src/introspection.rs b/apollo-router/src/introspection.rs index fad7d54823..33dd3d0394 100644 --- a/apollo-router/src/introspection.rs +++ b/apollo-router/src/introspection.rs @@ -25,7 +25,7 @@ impl Introspection { capacity: NonZeroUsize, ) -> Self { Self { - cache: CacheStorage::new(capacity, None, "introspection").await, + cache: CacheStorage::new(capacity, None, None, "introspection").await, planner, } } diff --git a/apollo-router/src/plugins/traffic_shaping/mod.rs b/apollo-router/src/plugins/traffic_shaping/mod.rs index a85bdce659..07ed0ccbf5 100644 --- a/apollo-router/src/plugins/traffic_shaping/mod.rs +++ b/apollo-router/src/plugins/traffic_shaping/mod.rs @@ -280,7 +280,17 @@ impl Plugin for TrafficShaping { .as_ref() .map(|cache| cache.urls.clone()) { - Some(RedisCacheStorage::new(urls, None).await?) + Some( + RedisCacheStorage::new( + urls, + None, + init.config + .experimental_cache + .as_ref() + .and_then(|c| c.timeout), + ) + .await?, + ) } else { None }; diff --git a/docs/source/configuration/distributed-caching.mdx b/docs/source/configuration/distributed-caching.mdx index 7f0f388036..654371d2b3 100644 --- a/docs/source/configuration/distributed-caching.mdx +++ b/docs/source/configuration/distributed-caching.mdx @@ -88,6 +88,7 @@ supergraph: experimental_cache: redis: #highlight-line urls: ["redis://..."] #highlight-line + timeout: 5ms # Optional, by default: 2ms ``` The value of `urls` is a list of URLs for all Redis instances in your cluster. @@ -104,6 +105,7 @@ apq: cache: redis: #highlight-line urls: ["redis://..."] #highlight-line + timeout: 5ms # Optional, by default: 2ms ``` The value of `urls` is a list of URLs for all Redis instances in your cluster. From 8d9cedf080fd16b3aa7103cb9e5db5fda9dc2652 Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Mon, 18 Sep 2023 11:58:55 +0200 Subject: [PATCH 21/51] Activate query planner cache warm up by default (#3801) Fix #3766 Cache warm up has been in testing for long enough, and has proven highly effective in smoothing out latencies when updating a schema. So this will activate it by default. It can still be deactivated by configuring it to 0: ```yaml supergraph: query_planning: warmed_up_queries: 0 ``` By default, it will now warm up the cache with the 30% most used queries --- apollo-router/src/configuration/mod.rs | 9 ++++--- ...nfiguration__tests__schema_generation.snap | 11 ++++---- .../query_planner/caching_query_planner.rs | 3 ++- apollo-router/src/router_factory.rs | 26 +++++++++---------- apollo-router/src/services/router_service.rs | 2 +- .../src/services/supergraph_service.rs | 2 +- 6 files changed, 27 insertions(+), 26 deletions(-) diff --git a/apollo-router/src/configuration/mod.rs b/apollo-router/src/configuration/mod.rs index 4126a040ee..4aaa11c1c9 100644 --- a/apollo-router/src/configuration/mod.rs +++ b/apollo-router/src/configuration/mod.rs @@ -826,11 +826,12 @@ impl Default for Apq { pub(crate) struct QueryPlanning { /// Cache configuration pub(crate) experimental_cache: Cache, - /// Warm up the cache on reloads by running the query plan over - /// a list of the most used queries - /// Defaults to 0 (do not warm up the cache) + /// Warms up the cache on reloads by running the query plan over + /// a list of the most used queries (from the in memory cache) + /// Configures the number of queries warmed up. Defaults to 1/3 of + /// the in memory cache #[serde(default)] - pub(crate) warmed_up_queries: usize, + pub(crate) warmed_up_queries: Option, } /// Cache configuration diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 541dbd3a7a..df03391cf4 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -1907,7 +1907,7 @@ expression: "&schema" }, "redis": null }, - "warmed_up_queries": 0 + "warmed_up_queries": null } }, "type": "object", @@ -1956,7 +1956,7 @@ expression: "&schema" }, "redis": null }, - "warmed_up_queries": 0 + "warmed_up_queries": null }, "type": "object", "properties": { @@ -2019,11 +2019,12 @@ expression: "&schema" "additionalProperties": false }, "warmed_up_queries": { - "description": "Warm up the cache on reloads by running the query plan over a list of the most used queries Defaults to 0 (do not warm up the cache)", - "default": 0, + "description": "Warms up the cache on reloads by running the query plan over a list of the most used queries (from the in memory cache) Configures the number of queries warmed up. Defaults to 1/3 of the in memory cache", + "default": null, "type": "integer", "format": "uint", - "minimum": 0.0 + "minimum": 0.0, + "nullable": true } }, "additionalProperties": false diff --git a/apollo-router/src/query_planner/caching_query_planner.rs b/apollo-router/src/query_planner/caching_query_planner.rs index 52876b6e0e..36617410c0 100644 --- a/apollo-router/src/query_planner/caching_query_planner.rs +++ b/apollo-router/src/query_planner/caching_query_planner.rs @@ -91,8 +91,9 @@ where } } - pub(crate) async fn cache_keys(&self, count: usize) -> Vec { + pub(crate) async fn cache_keys(&self, count: Option) -> Vec { let keys = self.cache.in_memory_keys().await; + let count = count.unwrap_or(keys.len() / 3); keys.into_iter() .take(count) .map(|key| WarmUpCachingQueryKey { diff --git a/apollo-router/src/router_factory.rs b/apollo-router/src/router_factory.rs index 40f14b5155..0a254e598b 100644 --- a/apollo-router/src/router_factory.rs +++ b/apollo-router/src/router_factory.rs @@ -202,21 +202,19 @@ impl RouterSuperServiceFactory for YamlRouterFactory { QueryAnalysisLayer::new(supergraph_creator.schema(), Arc::clone(&configuration)).await; if let Some(previous_router) = previous_router { - if configuration.supergraph.query_planning.warmed_up_queries > 0 { - let cache_keys = previous_router - .cache_keys(configuration.supergraph.query_planning.warmed_up_queries) + let cache_keys = previous_router + .cache_keys(configuration.supergraph.query_planning.warmed_up_queries) + .await; + + if !cache_keys.is_empty() { + tracing::info!( + "warming up the query plan cache with {} queries, this might take a while", + cache_keys.len() + ); + + supergraph_creator + .warm_up_query_planner(&query_parsing_layer, cache_keys) .await; - - if !cache_keys.is_empty() { - tracing::info!( - "warming up the query plan cache with {} queries, this might take a while", - cache_keys.len() - ); - - supergraph_creator - .warm_up_query_planner(&query_parsing_layer, cache_keys) - .await; - } } }; diff --git a/apollo-router/src/services/router_service.rs b/apollo-router/src/services/router_service.rs index 25c27a8b25..ed8ef6dbf5 100644 --- a/apollo-router/src/services/router_service.rs +++ b/apollo-router/src/services/router_service.rs @@ -552,7 +552,7 @@ impl RouterCreator { } impl RouterCreator { - pub(crate) async fn cache_keys(&self, count: usize) -> Vec { + pub(crate) async fn cache_keys(&self, count: Option) -> Vec { self.supergraph_creator.cache_keys(count).await } diff --git a/apollo-router/src/services/supergraph_service.rs b/apollo-router/src/services/supergraph_service.rs index 198b29b8be..378ad77221 100644 --- a/apollo-router/src/services/supergraph_service.rs +++ b/apollo-router/src/services/supergraph_service.rs @@ -752,7 +752,7 @@ impl SupergraphCreator { ) } - pub(crate) async fn cache_keys(&self, count: usize) -> Vec { + pub(crate) async fn cache_keys(&self, count: Option) -> Vec { self.query_planner_service.cache_keys(count).await } From 871e13e82be702bfe6472b46def1d5531cf5afe8 Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Mon, 18 Sep 2023 12:25:25 +0200 Subject: [PATCH 22/51] randomize the query planner cache warm up (#3800) Fix #3769 When using query plan cache warm up, we take the list of most used queries from the in memory cache, and plan them. The list will come in order, from the most used to the least used. If multiple router instances are using cache warm up, and are also configured to use the distributed cache for query plans, then they would all plan the list of queries in the same order, since they will have roughly the same distribution in the in memory cache. By randomizing the list of queries, we increase the chances that other instances can get plans from the distributed cache during warm up instead of calculating them, because other instances might have already done the work. --- apollo-router/src/query_planner/caching_query_planner.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apollo-router/src/query_planner/caching_query_planner.rs b/apollo-router/src/query_planner/caching_query_planner.rs index 36617410c0..b5dc8f9eef 100644 --- a/apollo-router/src/query_planner/caching_query_planner.rs +++ b/apollo-router/src/query_planner/caching_query_planner.rs @@ -7,6 +7,8 @@ use apollo_compiler::InputDatabase; use futures::future::BoxFuture; use indexmap::IndexMap; use query_planner::QueryPlannerPlugin; +use rand::seq::SliceRandom; +use rand::thread_rng; use router_bridge::planner::Planner; use router_bridge::planner::UsageReporting; use sha2::Digest; @@ -107,7 +109,7 @@ where pub(crate) async fn warm_up( &mut self, query_analysis: &QueryAnalysisLayer, - cache_keys: Vec, + mut cache_keys: Vec, ) { let _timer = Timer::new(|duration| { ::tracing::info!( @@ -125,6 +127,8 @@ where }), ); + cache_keys.shuffle(&mut thread_rng()); + let mut count = 0usize; for WarmUpCachingQueryKey { mut query, From 1bc8f9bfa1498f9a91efd494442c9f43c427ef45 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 17:15:48 +0000 Subject: [PATCH 23/51] fix(deps): update rust crate clap to 4.4.4 --- Cargo.lock | 14 +++++++------- apollo-router-scaffold/Cargo.toml | 2 +- apollo-router/Cargo.toml | 2 +- xtask/Cargo.lock | 8 ++++---- xtask/Cargo.toml | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 687bc50e83..0823a8d0e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ dependencies = [ "buildstructor 0.5.3", "bytes", "ci_info", - "clap 4.4.3", + "clap 4.4.4", "console-subscriber", "dashmap", "derivative", @@ -430,7 +430,7 @@ version = "1.30.0" dependencies = [ "anyhow", "cargo-scaffold", - "clap 4.4.3", + "clap 4.4.4", "copy_dir", "regex", "str_inflector", @@ -1347,9 +1347,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.3" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive", @@ -1357,9 +1357,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", @@ -1667,7 +1667,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.3", + "clap 4.4.4", "criterion-plot", "futures", "is-terminal", diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml index a601871a01..881dd43dcc 100644 --- a/apollo-router-scaffold/Cargo.toml +++ b/apollo-router-scaffold/Cargo.toml @@ -8,7 +8,7 @@ publish = false [dependencies] anyhow = "1.0.75" -clap = { version = "4.4.3", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } cargo-scaffold = { version = "0.8.10", default-features = false } regex = "1" str_inflector = "0.12.0" diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index f1082d64bd..e18268e438 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -69,7 +69,7 @@ base64 = "0.21.2" bloomfilter = "1.0.12" buildstructor = "0.5.3" bytes = "1.5.0" -clap = { version = "4.4.3", default-features = false, features = [ +clap = { version = "4.4.4", default-features = false, features = [ "env", "derive", "std", diff --git a/xtask/Cargo.lock b/xtask/Cargo.lock index 188b868d76..f2b6342c9e 100644 --- a/xtask/Cargo.lock +++ b/xtask/Cargo.lock @@ -228,9 +228,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.3" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive", @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 7e8f6b3c77..ad36d2b3b5 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -11,7 +11,7 @@ publish = false [dependencies] anyhow = "1" camino = "1" -clap = { version = "4.4.3", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } cargo_metadata = "0.17" # Only use the `clock` features of `chrono` to avoid the `time` dependency # impacted by CVE-2020-26235. https://github.com/chronotope/chrono/issues/602 From 82b168874859ba574bb7ce47743aa220c3e5db52 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 07:56:23 +0000 Subject: [PATCH 24/51] fix(deps): update dependency graphql to v16.8.1 --- .../tracing/datadog-subgraph/package-lock.json | 6 +++--- .../tracing/jaeger-subgraph/package-lock.json | 12 ++++++------ .../tracing/zipkin-subgraph/package-lock.json | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/dockerfiles/tracing/datadog-subgraph/package-lock.json b/dockerfiles/tracing/datadog-subgraph/package-lock.json index cfa64b06c1..786c186ee5 100644 --- a/dockerfiles/tracing/datadog-subgraph/package-lock.json +++ b/dockerfiles/tracing/datadog-subgraph/package-lock.json @@ -1144,9 +1144,9 @@ } }, "node_modules/graphql": { - "version": "16.8.0", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.0.tgz", - "integrity": "sha512-0oKGaR+y3qcS5mCu1vb7KG+a89vjn06C7Ihq/dDl3jA+A8B3TKomvi3CiEcVLJQGalbu8F52LxkOym7U5sSfbg==", + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } diff --git a/dockerfiles/tracing/jaeger-subgraph/package-lock.json b/dockerfiles/tracing/jaeger-subgraph/package-lock.json index 89c196fbb3..8a6aae61dd 100644 --- a/dockerfiles/tracing/jaeger-subgraph/package-lock.json +++ b/dockerfiles/tracing/jaeger-subgraph/package-lock.json @@ -893,9 +893,9 @@ } }, "node_modules/graphql": { - "version": "16.8.0", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.0.tgz", - "integrity": "sha512-0oKGaR+y3qcS5mCu1vb7KG+a89vjn06C7Ihq/dDl3jA+A8B3TKomvi3CiEcVLJQGalbu8F52LxkOym7U5sSfbg==", + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -2212,9 +2212,9 @@ } }, "graphql": { - "version": "16.8.0", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.0.tgz", - "integrity": "sha512-0oKGaR+y3qcS5mCu1vb7KG+a89vjn06C7Ihq/dDl3jA+A8B3TKomvi3CiEcVLJQGalbu8F52LxkOym7U5sSfbg==" + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==" }, "graphql-tag": { "version": "2.12.6", diff --git a/dockerfiles/tracing/zipkin-subgraph/package-lock.json b/dockerfiles/tracing/zipkin-subgraph/package-lock.json index 80a842008c..8ce9ded9c1 100644 --- a/dockerfiles/tracing/zipkin-subgraph/package-lock.json +++ b/dockerfiles/tracing/zipkin-subgraph/package-lock.json @@ -914,9 +914,9 @@ } }, "node_modules/graphql": { - "version": "16.8.0", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.0.tgz", - "integrity": "sha512-0oKGaR+y3qcS5mCu1vb7KG+a89vjn06C7Ihq/dDl3jA+A8B3TKomvi3CiEcVLJQGalbu8F52LxkOym7U5sSfbg==", + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -2277,9 +2277,9 @@ } }, "graphql": { - "version": "16.8.0", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.0.tgz", - "integrity": "sha512-0oKGaR+y3qcS5mCu1vb7KG+a89vjn06C7Ihq/dDl3jA+A8B3TKomvi3CiEcVLJQGalbu8F52LxkOym7U5sSfbg==" + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==" }, "graphql-tag": { "version": "2.12.6", From f16523cd7a3f97f8af3c47fb2237622f8c43fb39 Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Tue, 19 Sep 2023 13:43:28 +0200 Subject: [PATCH 25/51] warm up the query plan cache with persisted queries (#3829) Fix #3770 --- apollo-router/src/axum_factory/tests.rs | 2 ++ .../src/plugins/include_subgraph_errors.rs | 2 ++ .../src/plugins/traffic_shaping/mod.rs | 2 ++ .../query_planner/caching_query_planner.rs | 31 ++++++++++++++++++- apollo-router/src/router_factory.rs | 21 ++++++------- .../persisted_queries/manifest_poller.rs | 8 +++++ .../services/layers/persisted_queries/mod.rs | 6 ++++ apollo-router/src/services/router_service.rs | 5 +-- .../src/services/supergraph_service.rs | 4 ++- apollo-router/src/test_harness.rs | 3 ++ 10 files changed, 68 insertions(+), 16 deletions(-) diff --git a/apollo-router/src/axum_factory/tests.rs b/apollo-router/src/axum_factory/tests.rs index 3abf29f696..a918a8387b 100644 --- a/apollo-router/src/axum_factory/tests.rs +++ b/apollo-router/src/axum_factory/tests.rs @@ -68,6 +68,7 @@ use crate::query_planner::BridgeQueryPlanner; use crate::router_factory::create_plugins; use crate::router_factory::Endpoint; use crate::router_factory::RouterFactory; +use crate::services::layers::persisted_queries::PersistedQueryLayer; use crate::services::layers::query_analysis::QueryAnalysisLayer; use crate::services::layers::static_page::home_page_content; use crate::services::layers::static_page::sandbox_page_content; @@ -2318,6 +2319,7 @@ async fn test_supergraph_timeout() { let service = RouterCreator::new( QueryAnalysisLayer::new(supergraph_creator.schema(), Arc::clone(&conf)).await, + Arc::new(PersistedQueryLayer::new(&conf).await.unwrap()), Arc::new(supergraph_creator), conf.clone(), ) diff --git a/apollo-router/src/plugins/include_subgraph_errors.rs b/apollo-router/src/plugins/include_subgraph_errors.rs index b41f25361e..8f66888243 100644 --- a/apollo-router/src/plugins/include_subgraph_errors.rs +++ b/apollo-router/src/plugins/include_subgraph_errors.rs @@ -92,6 +92,7 @@ mod test { use crate::plugin::DynPlugin; use crate::query_planner::BridgeQueryPlanner; use crate::router_factory::create_plugins; + use crate::services::layers::persisted_queries::PersistedQueryLayer; use crate::services::layers::query_analysis::QueryAnalysisLayer; use crate::services::router; use crate::services::router_service::RouterCreator; @@ -214,6 +215,7 @@ mod test { RouterCreator::new( QueryAnalysisLayer::new(supergraph_creator.schema(), Default::default()).await, + Arc::new(PersistedQueryLayer::new(&Default::default()).await.unwrap()), Arc::new(supergraph_creator), Arc::new(Configuration::default()), ) diff --git a/apollo-router/src/plugins/traffic_shaping/mod.rs b/apollo-router/src/plugins/traffic_shaping/mod.rs index 07ed0ccbf5..6d21338d73 100644 --- a/apollo-router/src/plugins/traffic_shaping/mod.rs +++ b/apollo-router/src/plugins/traffic_shaping/mod.rs @@ -488,6 +488,7 @@ mod test { use crate::plugin::DynPlugin; use crate::query_planner::BridgeQueryPlanner; use crate::router_factory::create_plugins; + use crate::services::layers::persisted_queries::PersistedQueryLayer; use crate::services::layers::query_analysis::QueryAnalysisLayer; use crate::services::router; use crate::services::router_service::RouterCreator; @@ -608,6 +609,7 @@ mod test { RouterCreator::new( QueryAnalysisLayer::new(supergraph_creator.schema(), Default::default()).await, + Arc::new(PersistedQueryLayer::new(&Default::default()).await.unwrap()), Arc::new(supergraph_creator), Arc::new(Configuration::default()), ) diff --git a/apollo-router/src/query_planner/caching_query_planner.rs b/apollo-router/src/query_planner/caching_query_planner.rs index b5dc8f9eef..18e20d387a 100644 --- a/apollo-router/src/query_planner/caching_query_planner.rs +++ b/apollo-router/src/query_planner/caching_query_planner.rs @@ -28,6 +28,7 @@ use crate::plugins::telemetry::utils::Timer; use crate::query_planner::labeler::add_defer_labels; use crate::query_planner::BridgeQueryPlanner; use crate::query_planner::QueryPlanResult; +use crate::services::layers::persisted_queries::PersistedQueryLayer; use crate::services::layers::query_analysis::Compiler; use crate::services::layers::query_analysis::QueryAnalysisLayer; use crate::services::query_planner; @@ -109,6 +110,7 @@ where pub(crate) async fn warm_up( &mut self, query_analysis: &QueryAnalysisLayer, + persisted_query_layer: &PersistedQueryLayer, mut cache_keys: Vec, ) { let _timer = Timer::new(|duration| { @@ -129,12 +131,39 @@ where cache_keys.shuffle(&mut thread_rng()); + let persisted_queries_operations = persisted_query_layer.all_operations(); + + let capacity = cache_keys.len() + + persisted_queries_operations + .as_ref() + .map(|ops| ops.len()) + .unwrap_or(0); + tracing::info!( + "warming up the query plan cache with {} queries, this might take a while", + capacity + ); + + // persisted queries are added first because they should get a lower priority in the LRU cache, + // since a lot of them may be there to support old clients + let mut all_cache_keys = Vec::with_capacity(capacity); + if let Some(queries) = persisted_queries_operations { + for query in queries { + all_cache_keys.push(WarmUpCachingQueryKey { + query, + operation: None, + metadata: CacheKeyMetadata::default(), + }); + } + } + + all_cache_keys.extend(cache_keys.into_iter()); + let mut count = 0usize; for WarmUpCachingQueryKey { mut query, operation, metadata, - } in cache_keys + } in all_cache_keys { let caching_key = CachingQueryKey { schema_id: schema_id.clone(), diff --git a/apollo-router/src/router_factory.rs b/apollo-router/src/router_factory.rs index 0a254e598b..62dc5d888e 100644 --- a/apollo-router/src/router_factory.rs +++ b/apollo-router/src/router_factory.rs @@ -34,6 +34,7 @@ use crate::plugins::traffic_shaping::RetryPolicy; use crate::plugins::traffic_shaping::TrafficShaping; use crate::plugins::traffic_shaping::APOLLO_TRAFFIC_SHAPING; use crate::query_planner::BridgeQueryPlanner; +use crate::services::layers::persisted_queries::PersistedQueryLayer; use crate::services::layers::query_analysis::QueryAnalysisLayer; use crate::services::new_service::ServiceFactory; use crate::services::router; @@ -198,28 +199,24 @@ impl RouterSuperServiceFactory for YamlRouterFactory { let mut supergraph_creator = builder.build().await?; // Instantiate the parser here so we can use it to warm up the planner below - let query_parsing_layer = + let query_analysis_layer = QueryAnalysisLayer::new(supergraph_creator.schema(), Arc::clone(&configuration)).await; + let persisted_query_layer = Arc::new(PersistedQueryLayer::new(&configuration).await?); + if let Some(previous_router) = previous_router { let cache_keys = previous_router .cache_keys(configuration.supergraph.query_planning.warmed_up_queries) .await; - if !cache_keys.is_empty() { - tracing::info!( - "warming up the query plan cache with {} queries, this might take a while", - cache_keys.len() - ); - - supergraph_creator - .warm_up_query_planner(&query_parsing_layer, cache_keys) - .await; - } + supergraph_creator + .warm_up_query_planner(&query_analysis_layer, &persisted_query_layer, cache_keys) + .await; }; Ok(Self::RouterFactory::new( - query_parsing_layer, + query_analysis_layer, + persisted_query_layer, Arc::new(supergraph_creator), configuration, ) diff --git a/apollo-router/src/services/layers/persisted_queries/manifest_poller.rs b/apollo-router/src/services/layers/persisted_queries/manifest_poller.rs index 0af6df15d5..552536a18c 100644 --- a/apollo-router/src/services/layers/persisted_queries/manifest_poller.rs +++ b/apollo-router/src/services/layers/persisted_queries/manifest_poller.rs @@ -285,6 +285,14 @@ impl PersistedQueryManifestPoller { .cloned() } + pub(crate) fn get_all_operations(&self) -> Vec { + let state = self + .state + .read() + .expect("could not acquire read lock on persisted query manifest state"); + state.persisted_query_manifest.values().cloned().collect() + } + pub(crate) fn action_for_freeform_graphql( &self, query: &str, diff --git a/apollo-router/src/services/layers/persisted_queries/mod.rs b/apollo-router/src/services/layers/persisted_queries/mod.rs index 4250daabb3..025fc34613 100644 --- a/apollo-router/src/services/layers/persisted_queries/mod.rs +++ b/apollo-router/src/services/layers/persisted_queries/mod.rs @@ -252,6 +252,12 @@ impl PersistedQueryLayer { } } } + + pub(crate) fn all_operations(&self) -> Option> { + self.manifest_poller + .as_ref() + .map(|poller| poller.get_all_operations()) + } } fn log_unknown_operation(operation_body: &str) { diff --git a/apollo-router/src/services/router_service.rs b/apollo-router/src/services/router_service.rs index ed8ef6dbf5..8adbf23041 100644 --- a/apollo-router/src/services/router_service.rs +++ b/apollo-router/src/services/router_service.rs @@ -123,6 +123,7 @@ pub(crate) async fn from_supergraph_mock_callback_and_configuration( RouterCreator::new( QueryAnalysisLayer::new(supergraph_creator.schema(), Arc::clone(&configuration)).await, + Arc::new(PersistedQueryLayer::new(&configuration).await.unwrap()), Arc::new(supergraph_creator), configuration, ) @@ -172,6 +173,7 @@ pub(crate) async fn empty() -> impl Service< RouterCreator::new( QueryAnalysisLayer::new(supergraph_creator.schema(), Default::default()).await, + Arc::new(PersistedQueryLayer::new(&Default::default()).await.unwrap()), Arc::new(supergraph_creator), Arc::new(Configuration::default()), ) @@ -496,6 +498,7 @@ impl RouterFactory for RouterCreator { impl RouterCreator { pub(crate) async fn new( query_analysis_layer: QueryAnalysisLayer, + persisted_query_layer: Arc, supergraph_creator: Arc, configuration: Arc, ) -> Result { @@ -509,8 +512,6 @@ impl RouterCreator { APQLayer::disabled() }; - let persisted_query_layer = Arc::new(PersistedQueryLayer::new(&configuration).await?); - Ok(Self { supergraph_creator, static_page, diff --git a/apollo-router/src/services/supergraph_service.rs b/apollo-router/src/services/supergraph_service.rs index 378ad77221..5bf5ce0159 100644 --- a/apollo-router/src/services/supergraph_service.rs +++ b/apollo-router/src/services/supergraph_service.rs @@ -28,6 +28,7 @@ use tracing_futures::Instrument; use super::execution::QueryPlan; use super::layers::allow_only_http_post_mutations::AllowOnlyHttpPostMutationsLayer; use super::layers::content_negotiation; +use super::layers::persisted_queries::PersistedQueryLayer; use super::layers::query_analysis::Compiler; use super::layers::query_analysis::QueryAnalysisLayer; use super::new_service::ServiceFactory; @@ -763,10 +764,11 @@ impl SupergraphCreator { pub(crate) async fn warm_up_query_planner( &mut self, query_parser: &QueryAnalysisLayer, + persisted_query_layer: &PersistedQueryLayer, cache_keys: Vec, ) { self.query_planner_service - .warm_up(query_parser, cache_keys) + .warm_up(query_parser, persisted_query_layer, cache_keys) .await } } diff --git a/apollo-router/src/test_harness.rs b/apollo-router/src/test_harness.rs index 237d905cfb..189baf39f5 100644 --- a/apollo-router/src/test_harness.rs +++ b/apollo-router/src/test_harness.rs @@ -20,6 +20,7 @@ use crate::plugin::PluginInit; use crate::plugins::telemetry::reload::init_telemetry; use crate::router_factory::YamlRouterFactory; use crate::services::execution; +use crate::services::layers::persisted_queries::PersistedQueryLayer; use crate::services::layers::query_analysis::QueryAnalysisLayer; use crate::services::router; use crate::services::router_service::RouterCreator; @@ -272,6 +273,7 @@ impl<'a> TestHarness<'a> { let (config, supergraph_creator) = self.build_common().await?; let router_creator = RouterCreator::new( QueryAnalysisLayer::new(supergraph_creator.schema(), Arc::clone(&config)).await, + Arc::new(PersistedQueryLayer::new(&config).await.unwrap()), Arc::new(supergraph_creator), config, ) @@ -296,6 +298,7 @@ impl<'a> TestHarness<'a> { let (config, supergraph_creator) = self.build_common().await?; let router_creator = RouterCreator::new( QueryAnalysisLayer::new(supergraph_creator.schema(), Arc::clone(&config)).await, + Arc::new(PersistedQueryLayer::new(&config).await.unwrap()), Arc::new(supergraph_creator), config.clone(), ) From 8a208c9bda1b158a5808ac7232b7194bc98415ba Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 16:28:03 +0200 Subject: [PATCH 26/51] fix(deps): update apollo-rs crates (patch) (#3850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![Mend Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [apollo-compiler](https://togithub.com/apollographql/apollo-rs) | dependencies | patch | `0.11.1` -> `0.11.2` | | [apollo-parser](https://togithub.com/apollographql/apollo-rs) | dependencies | patch | `0.6.1` -> `0.6.2` | | [apollo-parser](https://togithub.com/apollographql/apollo-rs) | build-dependencies | patch | `0.6.1` -> `0.6.2` | --- ### Release Notes
apollographql/apollo-rs (apollo-compiler) ### [`v0.11.2`](https://togithub.com/apollographql/apollo-rs/releases/tag/apollo-compiler%400.11.2) [Compare Source](https://togithub.com/apollographql/apollo-rs/compare/apollo-compiler@0.11.1...apollo-compiler@0.11.2) #### Fixes - validate input value types, by \[goto-bus-stop] in \[pull/642] This fixes an oversight in the validation rules implemented by `compiler.db.validate()`. Previously, incorrect types on input values and arguments were not reported: ```graphql type ObjectType { id: ID! } input InputObject { ``` ### accepted in <= 0.11.1, reports "TypeThatDoesNotExist is not in scope" in 0.11.2 property: TypeThatDoesNotExist ### accepted in <= 0.11.1, reports "ObjectType is not an input type" in 0.11.2 inputType: ObjectType } ``` [goto-bus-stop]: https://togithub.com/goto-bus-stop [pull/642]: https://togithub.com/apollographql/apollo-rs/pull/642 ```
--- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://togithub.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Mend Renovate](https://www.mend.io/free-developer-tools/renovate/). View repository job log [here](https://developer.mend.io/github/apollographql/router). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 25 +++++++++++++------------ apollo-router-benchmarks/Cargo.toml | 2 +- apollo-router/Cargo.toml | 4 ++-- fuzz/Cargo.toml | 2 +- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0823a8d0e2..43e455c78f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,11 +216,11 @@ dependencies = [ [[package]] name = "apollo-compiler" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170c0c3761b0f11b9b0bd51b9720176776e1004df9f3b190d53a4f455873bcc" +checksum = "d9c5a0aa9cc4b5f9fc4aec3ecd76f4acc4407922c3c0e862b87f1c761c96a9bf" dependencies = [ - "apollo-parser 0.6.1", + "apollo-parser 0.6.2", "ariadne", "indexmap 2.0.0", "ordered-float 3.9.0", @@ -236,8 +236,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b323dfc43b95a8806a401aed913e84db297162c6295bca620c0f57956cdb52cf" dependencies = [ - "apollo-compiler 0.11.1", - "apollo-parser 0.6.1", + "apollo-compiler 0.11.2", + "apollo-parser 0.6.2", "thiserror", ] @@ -253,10 +253,11 @@ dependencies = [ [[package]] name = "apollo-parser" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d7e4a7f8862c19d5154b4a146f59d26a5f86678acd4fec1d5d87bd4c9284ab" +checksum = "e565c10133fbefa3cc9ac6c940bbfc2ed6086def9ec39b0f87a86cf9499aadd2" dependencies = [ + "memchr", "rowan", "thiserror", ] @@ -267,9 +268,9 @@ version = "1.30.0" dependencies = [ "access-json", "anyhow", - "apollo-compiler 0.11.1", + "apollo-compiler 0.11.2", "apollo-encoder", - "apollo-parser 0.6.1", + "apollo-parser 0.6.2", "arc-swap", "askama", "async-compression", @@ -412,7 +413,7 @@ dependencies = [ name = "apollo-router-benchmarks" version = "1.30.0" dependencies = [ - "apollo-parser 0.6.1", + "apollo-parser 0.6.2", "apollo-router", "apollo-smith", "arbitrary", @@ -445,7 +446,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13ed94bb9de9f24da12ca2122b8eeaa7484d17b090dc84daaaba6b6ac2bee89b" dependencies = [ "apollo-encoder", - "apollo-parser 0.6.1", + "apollo-parser 0.6.2", "arbitrary", "once_cell", "thiserror", @@ -5203,7 +5204,7 @@ dependencies = [ name = "router-fuzz" version = "0.0.0" dependencies = [ - "apollo-parser 0.6.1", + "apollo-parser 0.6.2", "apollo-smith", "env_logger", "libfuzzer-sys", diff --git a/apollo-router-benchmarks/Cargo.toml b/apollo-router-benchmarks/Cargo.toml index 31ed86c518..4aba6784ab 100644 --- a/apollo-router-benchmarks/Cargo.toml +++ b/apollo-router-benchmarks/Cargo.toml @@ -19,7 +19,7 @@ tower = "0.4" [build-dependencies] apollo-smith = { version = "0.4.0", features = ["parser-impl"] } -apollo-parser = "0.6.1" +apollo-parser = "0.6.2" arbitrary = "1.3.0" [[bench]] diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index e18268e438..9efcea4617 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -53,9 +53,9 @@ features = ["docs_rs"] askama = "0.12.0" access-json = "0.1.0" anyhow = "1.0.75" -apollo-compiler = "0.11.1" +apollo-compiler = "0.11.2" apollo-encoder = "0.7.0" -apollo-parser = "0.6.1" +apollo-parser = "0.6.2" arc-swap = "1.6.0" async-compression = { version = "0.4.1", features = [ "tokio", diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 6396f215d3..f95fe6ce39 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -12,7 +12,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" apollo-smith = { version = "0.4.0", features = ["parser-impl"] } -apollo-parser = "0.6.1" +apollo-parser = "0.6.2" env_logger = "0.10.0" log = "0.4" reqwest = { version = "0.11", default-features = false, features = ["json", "blocking"] } From 4d11a6ee532ab76910fd8d97d072d18bc95504ac Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 22:01:51 +0000 Subject: [PATCH 27/51] chore(deps): update rust crate fred to 6.3.2 --- Cargo.lock | 4 ++-- apollo-router/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43e455c78f..e5bcce7488 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2547,9 +2547,9 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "fred" -version = "6.3.1" +version = "6.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca2a979eaeb5d8a819edc193860ce54797730559464bc253cd3a2f765e58bd5" +checksum = "a15cc18b56395b8b15ffcdcea7fe8586e3a3ccb3d9dc3b9408800d9814efb08e" dependencies = [ "arc-swap", "arcstr", diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 9efcea4617..3a7d621301 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -88,7 +88,7 @@ diff = "0.1.13" directories = "5.0.1" displaydoc = "0.2" flate2 = "1.0.27" -fred = { version = "6.3.1", features = ["enable-rustls", "no-client-setname"] } +fred = { version = "6.3.2", features = ["enable-rustls", "no-client-setname"] } futures = { version = "0.3.28", features = ["thread-pool"] } graphql_client = "0.13.0" hex = "0.4.3" @@ -260,7 +260,7 @@ axum = { version = "0.6.20", features = [ "ws", ] } ecdsa = { version = "0.15.1", features = ["signing", "pem", "pkcs8"] } -fred = { version = "6.3.1", features = ["enable-rustls", "no-client-setname"] } +fred = { version = "6.3.2", features = ["enable-rustls", "no-client-setname"] } futures-test = "0.3.28" insta = { version = "1.31.0", features = ["json", "redactions", "yaml"] } maplit = "1.0.2" From 0996d4f39b00e4bfcf536faff4c5687161db3887 Mon Sep 17 00:00:00 2001 From: Jesse Rosenberger Date: Wed, 20 Sep 2023 12:59:15 +0000 Subject: [PATCH 28/51] prep release: v1.31.0-alpha.0 --- Cargo.lock | 6 ++-- apollo-router-benchmarks/Cargo.toml | 2 +- apollo-router-scaffold/Cargo.toml | 2 +- .../templates/base/Cargo.toml | 2 +- .../templates/base/xtask/Cargo.toml | 2 +- apollo-router/Cargo.toml | 2 +- .../tracing/docker-compose.datadog.yml | 2 +- dockerfiles/tracing/docker-compose.jaeger.yml | 2 +- dockerfiles/tracing/docker-compose.zipkin.yml | 2 +- docs/source/containerization/docker.mdx | 2 +- docs/source/containerization/kubernetes.mdx | 28 +++++++++---------- helm/chart/router/Chart.yaml | 4 +-- helm/chart/router/README.md | 8 +++--- licenses.html | 3 +- scripts/install.sh | 2 +- 15 files changed, 34 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5bcce7488..8854e31c52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,7 +264,7 @@ dependencies = [ [[package]] name = "apollo-router" -version = "1.30.0" +version = "1.31.0-alpha.0" dependencies = [ "access-json", "anyhow", @@ -411,7 +411,7 @@ dependencies = [ [[package]] name = "apollo-router-benchmarks" -version = "1.30.0" +version = "1.31.0-alpha.0" dependencies = [ "apollo-parser 0.6.2", "apollo-router", @@ -427,7 +427,7 @@ dependencies = [ [[package]] name = "apollo-router-scaffold" -version = "1.30.0" +version = "1.31.0-alpha.0" dependencies = [ "anyhow", "cargo-scaffold", diff --git a/apollo-router-benchmarks/Cargo.toml b/apollo-router-benchmarks/Cargo.toml index 4aba6784ab..4620aa59f5 100644 --- a/apollo-router-benchmarks/Cargo.toml +++ b/apollo-router-benchmarks/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apollo-router-benchmarks" -version = "1.30.0" +version = "1.31.0-alpha.0" authors = ["Apollo Graph, Inc. "] edition = "2021" license = "Elastic-2.0" diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml index 881dd43dcc..d6ba4afe51 100644 --- a/apollo-router-scaffold/Cargo.toml +++ b/apollo-router-scaffold/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apollo-router-scaffold" -version = "1.30.0" +version = "1.31.0-alpha.0" authors = ["Apollo Graph, Inc. "] edition = "2021" license = "Elastic-2.0" diff --git a/apollo-router-scaffold/templates/base/Cargo.toml b/apollo-router-scaffold/templates/base/Cargo.toml index 27cb00b340..0f8247ae22 100644 --- a/apollo-router-scaffold/templates/base/Cargo.toml +++ b/apollo-router-scaffold/templates/base/Cargo.toml @@ -22,7 +22,7 @@ apollo-router = { path ="{{integration_test}}apollo-router" } apollo-router = { git="https://github.com/apollographql/router.git", branch="{{branch}}" } {{else}} # Note if you update these dependencies then also update xtask/Cargo.toml -apollo-router = "1.30.0" +apollo-router = "1.31.0-alpha.0" {{/if}} {{/if}} async-trait = "0.1.52" diff --git a/apollo-router-scaffold/templates/base/xtask/Cargo.toml b/apollo-router-scaffold/templates/base/xtask/Cargo.toml index fb2f1d2470..904ff767ca 100644 --- a/apollo-router-scaffold/templates/base/xtask/Cargo.toml +++ b/apollo-router-scaffold/templates/base/xtask/Cargo.toml @@ -13,7 +13,7 @@ apollo-router-scaffold = { path ="{{integration_test}}apollo-router-scaffold" } {{#if branch}} apollo-router-scaffold = { git="https://github.com/apollographql/router.git", branch="{{branch}}" } {{else}} -apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.30.0" } +apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.0" } {{/if}} {{/if}} anyhow = "1.0.58" diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 3a7d621301..962c2d881d 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apollo-router" -version = "1.30.0" +version = "1.31.0-alpha.0" authors = ["Apollo Graph, Inc. "] repository = "https://github.com/apollographql/router/" documentation = "https://docs.rs/apollo-router" diff --git a/dockerfiles/tracing/docker-compose.datadog.yml b/dockerfiles/tracing/docker-compose.datadog.yml index 1fe9e4facc..4d3f5ee0bc 100644 --- a/dockerfiles/tracing/docker-compose.datadog.yml +++ b/dockerfiles/tracing/docker-compose.datadog.yml @@ -3,7 +3,7 @@ services: apollo-router: container_name: apollo-router - image: ghcr.io/apollographql/router:v1.30.0 + image: ghcr.io/apollographql/router:v1.31.0-alpha.0 volumes: - ./supergraph.graphql:/etc/config/supergraph.graphql - ./router/datadog.router.yaml:/etc/config/configuration.yaml diff --git a/dockerfiles/tracing/docker-compose.jaeger.yml b/dockerfiles/tracing/docker-compose.jaeger.yml index 85230d7e3e..bb2226979f 100644 --- a/dockerfiles/tracing/docker-compose.jaeger.yml +++ b/dockerfiles/tracing/docker-compose.jaeger.yml @@ -4,7 +4,7 @@ services: apollo-router: container_name: apollo-router #build: ./router - image: ghcr.io/apollographql/router:v1.30.0 + image: ghcr.io/apollographql/router:v1.31.0-alpha.0 volumes: - ./supergraph.graphql:/etc/config/supergraph.graphql - ./router/jaeger.router.yaml:/etc/config/configuration.yaml diff --git a/dockerfiles/tracing/docker-compose.zipkin.yml b/dockerfiles/tracing/docker-compose.zipkin.yml index d31d365126..191b008688 100644 --- a/dockerfiles/tracing/docker-compose.zipkin.yml +++ b/dockerfiles/tracing/docker-compose.zipkin.yml @@ -4,7 +4,7 @@ services: apollo-router: container_name: apollo-router build: ./router - image: ghcr.io/apollographql/router:v1.30.0 + image: ghcr.io/apollographql/router:v1.31.0-alpha.0 volumes: - ./supergraph.graphql:/etc/config/supergraph.graphql - ./router/zipkin.router.yaml:/etc/config/configuration.yaml diff --git a/docs/source/containerization/docker.mdx b/docs/source/containerization/docker.mdx index 9ddd11a47a..78e852ea46 100644 --- a/docs/source/containerization/docker.mdx +++ b/docs/source/containerization/docker.mdx @@ -11,7 +11,7 @@ The default behaviour of the router images is suitable for a quickstart or devel Note: The [docker documentation](https://docs.docker.com/engine/reference/run/) for the run command may be helpful when reading through the examples. -Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.30.0` +Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.0` ## Override the configuration diff --git a/docs/source/containerization/kubernetes.mdx b/docs/source/containerization/kubernetes.mdx index 68a0fea56b..0b82e0d26a 100644 --- a/docs/source/containerization/kubernetes.mdx +++ b/docs/source/containerization/kubernetes.mdx @@ -13,7 +13,7 @@ import { Link } from 'gatsby'; [Helm](https://helm.sh) is the package manager for kubernetes. -There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.30.0/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes. +There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.0/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes. In both the following examples, we are using helm to install the router: - into namespace "router-deploy" (create namespace if it doesn't exist) @@ -64,10 +64,10 @@ kind: ServiceAccount metadata: name: release-name-router labels: - helm.sh/chart: router-1.30.0 + helm.sh/chart: router-1.31.0-alpha.0 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.30.0" + app.kubernetes.io/version: "v1.31.0-alpha.0" app.kubernetes.io/managed-by: Helm --- # Source: router/templates/secret.yaml @@ -76,10 +76,10 @@ kind: Secret metadata: name: "release-name-router" labels: - helm.sh/chart: router-1.30.0 + helm.sh/chart: router-1.31.0-alpha.0 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.30.0" + app.kubernetes.io/version: "v1.31.0-alpha.0" app.kubernetes.io/managed-by: Helm data: managedFederationApiKey: "UkVEQUNURUQ=" @@ -90,10 +90,10 @@ kind: ConfigMap metadata: name: release-name-router labels: - helm.sh/chart: router-1.30.0 + helm.sh/chart: router-1.31.0-alpha.0 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.30.0" + app.kubernetes.io/version: "v1.31.0-alpha.0" app.kubernetes.io/managed-by: Helm data: configuration.yaml: | @@ -117,10 +117,10 @@ kind: Service metadata: name: release-name-router labels: - helm.sh/chart: router-1.30.0 + helm.sh/chart: router-1.31.0-alpha.0 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.30.0" + app.kubernetes.io/version: "v1.31.0-alpha.0" app.kubernetes.io/managed-by: Helm spec: type: ClusterIP @@ -143,10 +143,10 @@ kind: Deployment metadata: name: release-name-router labels: - helm.sh/chart: router-1.30.0 + helm.sh/chart: router-1.31.0-alpha.0 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.30.0" + app.kubernetes.io/version: "v1.31.0-alpha.0" app.kubernetes.io/managed-by: Helm annotations: @@ -174,7 +174,7 @@ spec: - name: router securityContext: {} - image: "ghcr.io/apollographql/router:v1.30.0" + image: "ghcr.io/apollographql/router:v1.31.0-alpha.0" imagePullPolicy: IfNotPresent args: - --hot-reload @@ -226,10 +226,10 @@ kind: Pod metadata: name: "release-name-router-test-connection" labels: - helm.sh/chart: router-1.30.0 + helm.sh/chart: router-1.31.0-alpha.0 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.30.0" + app.kubernetes.io/version: "v1.31.0-alpha.0" app.kubernetes.io/managed-by: Helm annotations: "helm.sh/hook": test diff --git a/helm/chart/router/Chart.yaml b/helm/chart/router/Chart.yaml index 07dbc45a6a..bea5fb7feb 100644 --- a/helm/chart/router/Chart.yaml +++ b/helm/chart/router/Chart.yaml @@ -20,10 +20,10 @@ type: application # so it matches the shape of our release process and release automation. # By proxy of that decision, this version uses SemVer 2.0.0, though the prefix # of "v" is not included. -version: 1.30.0 +version: 1.31.0-alpha.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.30.0" \ No newline at end of file +appVersion: "v1.31.0-alpha.0" \ No newline at end of file diff --git a/helm/chart/router/README.md b/helm/chart/router/README.md index d31ac65966..eae73cda3b 100644 --- a/helm/chart/router/README.md +++ b/helm/chart/router/README.md @@ -2,7 +2,7 @@ [router](https://github.com/apollographql/router) Rust Graph Routing runtime for Apollo Federation -![Version: 1.30.0](https://img.shields.io/badge/Version-1.30.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.30.0](https://img.shields.io/badge/AppVersion-v1.30.0-informational?style=flat-square) +![Version: 1.31.0-alpha.0](https://img.shields.io/badge/Version-1.31.0--alpha.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.0](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.0-informational?style=flat-square) ## Prerequisites @@ -11,7 +11,7 @@ ## Get Repo Info ```console -helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.30.0 +helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.0 ``` ## Install Chart @@ -19,7 +19,7 @@ helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.30.0 **Important:** only helm3 is supported ```console -helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.30.0 --values my-values.yaml +helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.0 --values my-values.yaml ``` _See [configuration](#configuration) below._ @@ -48,7 +48,7 @@ helm show values oci://ghcr.io/apollographql/helm-charts/router | extraEnvVars | list | `[]` | | | extraEnvVarsCM | string | `""` | | | extraEnvVarsSecret | string | `""` | | -| extraLabels | object | `{}` | A map of extra labels to apply to the router deploment and containers Example: extraLabels: label_one_name: "label_one_value" label_two_name: "label_two_value" | +| extraLabels | object | `{}` | A map of extra labels to apply to the resources created by this chart Example: extraLabels: label_one_name: "label_one_value" label_two_name: "label_two_value" | | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | | fullnameOverride | string | `""` | | diff --git a/licenses.html b/licenses.html index 28c09f483c..888f30ee94 100644 --- a/licenses.html +++ b/licenses.html @@ -1715,6 +1715,7 @@

Used by:

  • clap_derive
  • clap_lex
  • opentelemetry
    • +
  • opentelemetry-aws
  • opentelemetry-datadog
  • opentelemetry-http
  • opentelemetry-jaeger
    • @@ -7298,7 +7299,6 @@

    Used by:

  • try_match
  • try_match
  • tungstenite
    • -
  • tungstenite
  • typed-builder
  • typetag
  • typetag-impl
    • @@ -12811,7 +12811,6 @@

    MIT License

    Used by:

    Copyright (c) 2017 Daniel Abramov
 Copyright (c) 2017 Alexey Galakhov
diff --git a/scripts/install.sh b/scripts/install.sh
index 318035641b..4d9c0ead7e 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -11,7 +11,7 @@ BINARY_DOWNLOAD_PREFIX="https://github.com/apollographql/router/releases/downloa
 
 # Router version defined in apollo-router's Cargo.toml
 # Note: Change this line manually during the release steps.
-PACKAGE_VERSION="v1.30.0"
+PACKAGE_VERSION="v1.31.0-alpha.0"
 
 download_binary() {
     downloader --check

From 3fcb78de7c6444cd1f7fe4cca6533bcc6bf8a085 Mon Sep 17 00:00:00 2001
From: Geoffroy Couprie 
Date: Wed, 20 Sep 2023 17:02:13 +0200
Subject: [PATCH 29/51] TLS client authentication (#3794)

Fix #3414

The router now supports TLS client authentication when connecting to
subgraphs. It can be configured as follows:

```yaml
tls:
  subgraph:
    all:
      client_authentication:
        certificate_chain: ${file./path/to/certificate_chain.pem}
        key: ${file./path/to/key.pem}
    # if configuring for a specific subgraph:
    subgraphs:
      # subgraph name
      products:
        client_authentication:
          certificate_chain: ${file./path/to/certificate_chain.pem}
          key: ${file./path/to/key.pem}
```

This also adds tests for subgraph TLS connections:
* self signed server certificates
* root certificate authority override
* client authentication
---
 .changesets/feat_geal_subgraph_mtls.md        |  21 +
 apollo-router/src/configuration/metrics.rs    |  10 +
 apollo-router/src/configuration/mod.rs        |  38 +-
 ...etrics__test__metrics@tls.router.yaml.snap |  10 +
 ...nfiguration__tests__schema_generation.snap |  55 +-
 .../testdata/metrics/tls.router.yaml          | 106 ++++
 apollo-router/src/router_factory.rs           |  31 +-
 .../src/services/subgraph_service.rs          | 546 ++++++++++++++++--
 apollo-router/src/services/testdata/CA/ca.crt |  31 +
 apollo-router/src/services/testdata/CA/ca.key |  52 ++
 .../src/services/testdata/client.crt          |  33 ++
 .../src/services/testdata/client.csr          |  26 +
 .../src/services/testdata/client.ext          |   7 +
 .../src/services/testdata/client.key          |  52 ++
 .../src/services/testdata/server.crt          |  33 ++
 .../src/services/testdata/server.csr          |  26 +
 .../src/services/testdata/server.ext          |   6 +
 .../src/services/testdata/server.key          |  52 ++
 .../services/testdata/server_self_signed.crt  |  32 +
 .../services/testdata/server_self_signed.csr  |  26 +
 apollo-router/src/services/testdata/tls.md    |  82 +++
 docs/source/configuration/overview.mdx        |  22 +-
 22 files changed, 1216 insertions(+), 81 deletions(-)
 create mode 100644 .changesets/feat_geal_subgraph_mtls.md
 create mode 100644 apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@tls.router.yaml.snap
 create mode 100644 apollo-router/src/configuration/testdata/metrics/tls.router.yaml
 create mode 100644 apollo-router/src/services/testdata/CA/ca.crt
 create mode 100644 apollo-router/src/services/testdata/CA/ca.key
 create mode 100644 apollo-router/src/services/testdata/client.crt
 create mode 100644 apollo-router/src/services/testdata/client.csr
 create mode 100644 apollo-router/src/services/testdata/client.ext
 create mode 100644 apollo-router/src/services/testdata/client.key
 create mode 100644 apollo-router/src/services/testdata/server.crt
 create mode 100644 apollo-router/src/services/testdata/server.csr
 create mode 100644 apollo-router/src/services/testdata/server.ext
 create mode 100644 apollo-router/src/services/testdata/server.key
 create mode 100644 apollo-router/src/services/testdata/server_self_signed.crt
 create mode 100644 apollo-router/src/services/testdata/server_self_signed.csr
 create mode 100644 apollo-router/src/services/testdata/tls.md

diff --git a/.changesets/feat_geal_subgraph_mtls.md b/.changesets/feat_geal_subgraph_mtls.md
new file mode 100644
index 0000000000..a377ab52bf
--- /dev/null
+++ b/.changesets/feat_geal_subgraph_mtls.md
@@ -0,0 +1,21 @@
+### TLS client authentication for subgraph requests ([Issue #3414](https://github.com/apollographql/router/issues/3414))
+
+The router now supports TLS client authentication when connecting to subgraphs. It can be configured as follows:
+
+```yaml
+tls:
+  subgraph:
+    all:
+      client_authentication:
+        certificate_chain: ${file./path/to/certificate_chain.pem}
+        key: ${file./path/to/key.pem}
+    # if configuring for a specific subgraph:
+    subgraphs:
+      # subgraph name
+      products:
+        client_authentication:
+          certificate_chain: ${file./path/to/certificate_chain.pem}
+          key: ${file./path/to/key.pem}
+```
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3794
\ No newline at end of file
diff --git a/apollo-router/src/configuration/metrics.rs b/apollo-router/src/configuration/metrics.rs
index 1bd5db048a..20dd595598 100644
--- a/apollo-router/src/configuration/metrics.rs
+++ b/apollo-router/src/configuration/metrics.rs
@@ -260,6 +260,16 @@ impl Metrics {
             opt.subgraph,
             "$.subgraph..enabled[?(@ == true)]"
         );
+        log_usage_metrics!(
+            value.apollo.router.config.tls,
+            "$.tls",
+            opt.router.tls.server,
+            "$.supergraph",
+            opt.router.tls.subgraph.ca_override,
+            "$[?(@.subgraph..certificate_authorities)]",
+            opt.router.tls.subgraph.client_authentication,
+            "$.subgraph..client_authentication"
+        );
         log_usage_metrics!(
             value.apollo.router.config.traffic_shaping,
             "$.traffic_shaping",
diff --git a/apollo-router/src/configuration/mod.rs b/apollo-router/src/configuration/mod.rs
index 4aaa11c1c9..02c1694051 100644
--- a/apollo-router/src/configuration/mod.rs
+++ b/apollo-router/src/configuration/mod.rs
@@ -929,13 +929,11 @@ where
         .map_err(serde::de::Error::custom)
         .and_then(|mut certs| {
             if certs.len() > 1 {
-                Err(serde::de::Error::custom(
-                    "expected exactly one server certificate",
-                ))
+                Err(serde::de::Error::custom("expected exactly one certificate"))
             } else {
-                certs.pop().ok_or(serde::de::Error::custom(
-                    "expected exactly one server certificate",
-                ))
+                certs
+                    .pop()
+                    .ok_or(serde::de::Error::custom("expected exactly one certificate"))
             }
         })
 }
@@ -955,16 +953,16 @@ where
 {
     let data = String::deserialize(deserializer)?;
 
-    load_keys(&data).map_err(serde::de::Error::custom)
+    load_key(&data).map_err(serde::de::Error::custom)
 }
 
-fn load_certs(data: &str) -> io::Result> {
+pub(crate) fn load_certs(data: &str) -> io::Result> {
     certs(&mut BufReader::new(data.as_bytes()))
         .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid cert"))
         .map(|mut certs| certs.drain(..).map(Certificate).collect())
 }
 
-fn load_keys(data: &str) -> io::Result {
+pub(crate) fn load_key(data: &str) -> io::Result {
     let mut reader = BufReader::new(data.as_bytes());
     let mut key_iterator = iter::from_fn(|| read_one(&mut reader).transpose());
 
@@ -1008,14 +1006,20 @@ fn load_keys(data: &str) -> io::Result {
 pub(crate) struct TlsSubgraph {
     /// list of certificate authorities in PEM format
     pub(crate) certificate_authorities: Option,
+    /// client certificate authentication
+    pub(crate) client_authentication: Option,
 }
 
 #[buildstructor::buildstructor]
 impl TlsSubgraph {
     #[builder]
-    pub(crate) fn new(certificate_authorities: Option) -> Self {
+    pub(crate) fn new(
+        certificate_authorities: Option,
+        client_authentication: Option,
+    ) -> Self {
         Self {
             certificate_authorities,
+            client_authentication,
         }
     }
 }
@@ -1026,6 +1030,20 @@ impl Default for TlsSubgraph {
     }
 }
 
+/// TLS client authentication
+#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct TlsClientAuth {
+    /// list of certificates in PEM format
+    #[serde(deserialize_with = "deserialize_certificate_chain", skip_serializing)]
+    #[schemars(with = "String")]
+    pub(crate) certificate_chain: Vec,
+    /// key in PEM format
+    #[serde(deserialize_with = "deserialize_key", skip_serializing)]
+    #[schemars(with = "String")]
+    pub(crate) key: PrivateKey,
+}
+
 /// Configuration options pertaining to the sandbox page.
 #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
 #[serde(deny_unknown_fields)]
diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@tls.router.yaml.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@tls.router.yaml.snap
new file mode 100644
index 0000000000..5e9309469e
--- /dev/null
+++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@tls.router.yaml.snap
@@ -0,0 +1,10 @@
+---
+source: apollo-router/src/configuration/metrics.rs
+expression: "&metrics.metrics"
+---
+value.apollo.router.config.tls:
+  - 1
+  - opt__router__tls__server__: true
+    opt__router__tls__subgraph__ca_override__: true
+    opt__router__tls__subgraph__client_authentication__: true
+
diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap
index df03391cf4..428ef466d4 100644
--- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap
+++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap
@@ -5384,7 +5384,8 @@ expression: "&schema"
         "supergraph": null,
         "subgraph": {
           "all": {
-            "certificate_authorities": null
+            "certificate_authorities": null,
+            "client_authentication": null
           },
           "subgraphs": {}
         }
@@ -5395,7 +5396,8 @@ expression: "&schema"
           "description": "Configuration options pertaining to the subgraph server component.",
           "default": {
             "all": {
-              "certificate_authorities": null
+              "certificate_authorities": null,
+              "client_authentication": null
             },
             "subgraphs": {}
           },
@@ -5404,7 +5406,8 @@ expression: "&schema"
             "all": {
               "description": "options applying to all subgraphs",
               "default": {
-                "certificate_authorities": null
+                "certificate_authorities": null,
+                "client_authentication": null
               },
               "type": "object",
               "properties": {
@@ -5413,6 +5416,29 @@ expression: "&schema"
                   "default": null,
                   "type": "string",
                   "nullable": true
+                },
+                "client_authentication": {
+                  "description": "client certificate authentication",
+                  "default": null,
+                  "type": "object",
+                  "required": [
+                    "certificate_chain",
+                    "key"
+                  ],
+                  "properties": {
+                    "certificate_chain": {
+                      "description": "list of certificates in PEM format",
+                      "writeOnly": true,
+                      "type": "string"
+                    },
+                    "key": {
+                      "description": "key in PEM format",
+                      "writeOnly": true,
+                      "type": "string"
+                    }
+                  },
+                  "additionalProperties": false,
+                  "nullable": true
                 }
               },
               "additionalProperties": false
@@ -5430,6 +5456,29 @@ expression: "&schema"
                     "default": null,
                     "type": "string",
                     "nullable": true
+                  },
+                  "client_authentication": {
+                    "description": "client certificate authentication",
+                    "default": null,
+                    "type": "object",
+                    "required": [
+                      "certificate_chain",
+                      "key"
+                    ],
+                    "properties": {
+                      "certificate_chain": {
+                        "description": "list of certificates in PEM format",
+                        "writeOnly": true,
+                        "type": "string"
+                      },
+                      "key": {
+                        "description": "key in PEM format",
+                        "writeOnly": true,
+                        "type": "string"
+                      }
+                    },
+                    "additionalProperties": false,
+                    "nullable": true
                   }
                 },
                 "additionalProperties": false
diff --git a/apollo-router/src/configuration/testdata/metrics/tls.router.yaml b/apollo-router/src/configuration/testdata/metrics/tls.router.yaml
new file mode 100644
index 0000000000..00d1d3f63d
--- /dev/null
+++ b/apollo-router/src/configuration/testdata/metrics/tls.router.yaml
@@ -0,0 +1,106 @@
+tls:
+  supergraph:
+      # write the cert content directly because the test does not do expansion
+      certificate: &cert |
+        -----BEGIN CERTIFICATE-----
+        MIIF9TCCA92gAwIBAgIUM+6LSYfTRzSalYzqdFfuPbcznyswDQYJKoZIhvcNAQEL
+        BQAwQDELMAkGA1UEBhMCRlIxCjAIBgNVBAgMASAxCjAIBgNVBAoMASAxGTAXBgNV
+        BAMMEGxvY2FsLmFwb2xsby5kZXYwHhcNMjIxMjE2MTQ1OTMzWhcNMjMwMTE1MTQ1
+        OTMzWjBAMQswCQYDVQQGEwJGUjEKMAgGA1UECAwBIDEKMAgGA1UECgwBIDEZMBcG
+        A1UEAwwQbG9jYWwuYXBvbGxvLmRldjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
+        AgoCggIBALK52xtnhD1MJEuuXbLlEU3tcPO3MIWYFY2i+rTyYQYKxa5a4QG9vBjB
+        bQb18b2xXVxmCs57MYt9v5GQrU4Dc55qWXVzSFK3wLD8PvS+NaTkjh+TH1MbW8Rr
+        BVxTq1XD0HJAJfXdbTlg62VoKk6UXFk+YH/In+u1UAq0T1amC39B8hiTFNd2Yawg
+        SKn4i+6NmZluzIb88ZLzRb2xrnEd2FG4JAucPHpTjmNtwFzl3nmbgMNKntLA3Ac+
+        CdaIWuPqkbDEDzR5mP8tx2IzUSz3C08Z1Oo+8uS5aOyWg8l4MPBhyWONFA8ilvd3
+        +yjzPKwa/zFEozoUp5GWSWLl53Ff6anw54yUIND0qhD5X4ICtOk2F41Gwv/GKTSO
+        AnfwpxZiUji2OOZwXQ/Zs+lUXTgQvshvb6PXbJT6T3wxou+WpVJFDctELBNdMNbe
+        WldtYvPry7rngLWOUsLq6c/oQibvL19Pc98532LKsWFsYEMRVA7WNsyj040Y9FoO
+        zBgvZ/AyxgT/23/P9xJxg0RjqOkO8jPx5kpDOL9c8qkKds7CQ5z4d3DZuzLjfKUw
+        pT3u10an1nh5xmcSH9vLnZuDoJL9OzJdmQ8pEdKdQGTTP4KXM/OUh8+sSxFyLoYV
+        850SeydMTTm72GkWzwq2npp2KNo41F0mT2eyvQSNy0PIN6eSRgpnAgMBAAGjgeYw
+        geMwHQYDVR0OBBYEFDV0smlfWnSnE/PtxF65lwn5ewgrMHsGA1UdIwR0MHKAFDV0
+        smlfWnSnE/PtxF65lwn5ewgroUSkQjBAMQswCQYDVQQGEwJGUjEKMAgGA1UECAwB
+        IDEKMAgGA1UECgwBIDEZMBcGA1UEAwwQbG9jYWwuYXBvbGxvLmRldoIUM+6LSYfT
+        RzSalYzqdFfuPbcznyswCwYDVR0PBAQDAgL8MBsGA1UdEQQUMBKCEGxvY2FsLmFw
+        b2xsby5kZXYwGwYDVR0SBBQwEoIQbG9jYWwuYXBvbGxvLmRldjANBgkqhkiG9w0B
+        AQsFAAOCAgEAZtNEFzXO0phLUQm2yRZzOA5OPwsW97iMBUc2o5tP8jkkmWTNMWHe
+        1COAkPVBpPS+lbCAMdoMd+yrQ/tZdMmVvqXYMc087ZkIGeIG8NOHWJkBoAlV3mYP
+        feb8nbbZBHLzZUgj8p77sQeCR3gbodeUHoB3YEgb/btz39r6zYBdBcbrhU1D4Qki
+        +xpd1iYdo/qI9TwgnEavcIZ4Zpv7T6IvxPXQ6WjWofXlb3G8atm5lL88TxMszHv4
+        d2A3giMd4wv66usme9CN2kFKV568eQqnqAzY+bNGdAVlLX2ieWCKT9NmUhHc8b1M
+        oaS6E/qlcOT4c+F8/kDcW35TasPuzLEH8YBrn+e+rl0etv6DJL3gBqMciJNJ0DSj
+        YW0inRx6VaQCH0iqzeKjy7bas6Mj/emfkmMIuzL2UVFE2khfMqbpaR9Uat4jbIzH
+        Pfh5zF40bklOqA5axztJurKWv5deEof5PZ5jLx47VIU3VrwYmIUEUpOdEi426LwX
+        0TSEG0P8d82UqU+mh7Ibcd1KWTmmwA7pJ9hsN6n2VYhogojh1n1lwDH0g6ND6+mh
+        LOGdw2a3DeyWSZNl/HRezyq983gbK/1U2DeuoxzAC8axEJa4iRRBWMKX7XdBuuHD
+        wj3nI/0PXcNFsiaB7qPpIFCv7F9fw44tdh58mCdQSWC+JeJp43E9Wfg=
+        -----END CERTIFICATE-----
+      certificate_chain: *cert
+      key: &key |
+        -----BEGIN PRIVATE KEY-----
+        MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQCyudsbZ4Q9TCRL
+        rl2y5RFN7XDztzCFmBWNovq08mEGCsWuWuEBvbwYwW0G9fG9sV1cZgrOezGLfb+R
+        kK1OA3Oeall1c0hSt8Cw/D70vjWk5I4fkx9TG1vEawVcU6tVw9ByQCX13W05YOtl
+        aCpOlFxZPmB/yJ/rtVAKtE9Wpgt/QfIYkxTXdmGsIEip+IvujZmZbsyG/PGS80W9
+        sa5xHdhRuCQLnDx6U45jbcBc5d55m4DDSp7SwNwHPgnWiFrj6pGwxA80eZj/Lcdi
+        M1Es9wtPGdTqPvLkuWjsloPJeDDwYcljjRQPIpb3d/so8zysGv8xRKM6FKeRlkli
+        5edxX+mp8OeMlCDQ9KoQ+V+CArTpNheNRsL/xik0jgJ38KcWYlI4tjjmcF0P2bPp
+        VF04EL7Ib2+j12yU+k98MaLvlqVSRQ3LRCwTXTDW3lpXbWLz68u654C1jlLC6unP
+        6EIm7y9fT3PfOd9iyrFhbGBDEVQO1jbMo9ONGPRaDswYL2fwMsYE/9t/z/cScYNE
+        Y6jpDvIz8eZKQzi/XPKpCnbOwkOc+Hdw2bsy43ylMKU97tdGp9Z4ecZnEh/by52b
+        g6CS/TsyXZkPKRHSnUBk0z+ClzPzlIfPrEsRci6GFfOdEnsnTE05u9hpFs8Ktp6a
+        dijaONRdJk9nsr0EjctDyDenkkYKZwIDAQABAoICAAtyFZMStQhL6QMjvoJnYw1P
+        iG1DLQtRKwgwCMgvwYDmjbRVw5Ud9n7LXFUWyQ1x3128dzKz9v9M5UjIMCEP3Yam
+        nuYDpP0PIXr0HIAF8l+F94gUHuxukxjoFabNAOr0KFQ4wXWWYZlMGKcc3aC8pZFd
+        ikaEraElsmONGoudBJ14tq1WNf56aVThmGWyMhvr24tU6io25q2XgL7eMyKxW5oY
+        Jc7MiZ733OWHMMuCORYmnD9ldvheO3kHQxAHGXMBIaVlWOfuZZrp7pveV3N+uq2t
+        JNJ/h4SYTxzfor7zQIcUbBZBAajaeTqN53q+4QLQk8ku8RkWG4kaS8bWnFTJZKhO
+        75zg7UCFr+PiEiK2zZqMZEp8yiENu8PcdyEu8VME/GnCgBEEdFoi0zduUs45sE80
+        32DSmDhbaqxsN3wLNLLbOHd5Ewfjt2A9jh36OBCKn2l3FKyUr/Eh9GNjWpNlyN5H
+        kZ8PYh5DLfQUyLHJRpAFl2OZiA6IAPcduBRneG39WNk7L5F8gUTDMYcwOZXq8g/B
+        hGCMfqRcZ4Bwj0YrSME8S7NixhPokpk/997gfeHnsOMi3wIWBmxISyYXzn/Y23iC
+        +6RvtWD7WoC3a4zoMKwVqCrZFeQktWgBMJYWcip/47ijynlf2at5bWf6bWGNqzml
+        7vfJ1ijaT9YYrXdry+fhAoIBAQDqPq+wqaud8De4WJMb9aWux+N4h2A4A6j5LKKS
+        Dnfv1bIfqSx9w+gvPOSj/7T47VmauM4R15DRZBjYIwDuT14ExkDYfoIFnX/3XkaV
+        FGmt+mtro8dDZSt/aPcSVlFt5CAeYIokvfdBv3lAeANkUnjHWY6cO98HTb6DIGo0
+        b1sbuYeTgFSEhZjvaN97nYeGjPNxQy6lgL2ExOwvkLD+4X93JrITNAHwyprWBLZX
+        nMfp7R/Zn/pQ6n//IJZ+NdiEM3Scw6dOhLYQBny+r8l5K2HK48MDggPOrNe3YnJ0
+        6zkaAvrObIkj7eG0HlB4nwUKI5zfTbAWx5Nhft6PIbMZ4jN3AoIBAQDDUyzCpEFm
+        j0y4wtBYm3AOThWz2FTX9nsoz+CWntfDtObItfByPYDFKN3sbo62jtTtUiY6JIrs
+        Cy1AvbSMHD+GZ7RKtZ8U5Aosb+z1ddYEe/jHTncGCCK9sa8Rsw/bL4z+8WFXqo+B
+        H9IrAYb7XV3usb2TuJ0rXF5Dj7xnIIG0+ifnpCPo44VQMNFAE00/7v/AIDTgUJEf
+        ANqtqEhtI77lt/hEB4vK1Y0IpCNKkj56ZtNNnSAkS7mwV8nbmTp7RIgcMjdHj++s
+        yuPBGkk0oK03PDQbt7dqMqXax55x36fNwVavQnLOpS5utxmCPmHGZp8lmrZHe4e2
+        eSQ7IBMDYTyRAoIBAQDTNJBO9r0Rm+1xnxONSzEHZZ25KD7eYpZxjQhMLxV/Pyvr
+        FitSfliUdxAkusOfCssXEXhkZ/xggCNShkUpmpBIN8VyLqbnjzo5fVygwJYE010V
+        4cciAk91Atx7QS8MqXs4SI1mUY5mTtFyCoPsadwSyrImNmmC+VtEee6otprsZZ1T
+        posOLjNV8jZHDCONcvtxbUTa3ziCRNg8jva36fR3J4G6hNMXHGb9f6Q1XNx4FGD6
+        ZR9a5AVisSxgQgamNIr5agQpbMmHq4HAoVlEkpQLTs+gExOBvyCFbhOLTiffRz6C
+        7yO8LJmsQQUHrUHrAW9JfI/AClTdvHnJjnYhaW5DAoIBAQCf1/FJWCItTBf9G3Bl
+        Es8g5cXc56yHD666N2QT3umzvtceacXbt3kp38e9NLyVqU2W6SNfcYg+oubllFms
+        T3GtDDD+8qK89eFdfDrupP//q3RrpkrBJOdJVZ9vXJodRUydVevTUkEd6myTxSwx
+        iLbWH56ExQ/Z7D04DOihfHipIg6GAk1gyNDQTyLuzNzq9StWjwS2jTg1pv1OH+kl
+        Z5tRYrxI7+P2mcxQxgIbhJKcmIlTesJS8aWEKlOG4l55ghvg9zdF2QTK4z5/SIOg
+        Dd2y1hHOnQn8XnZcFAAWMHGicBYAVuCdO5BECpNVgreBJXoXzARfezgUnA6KVDU7
+        DtgBAoIBAQDIl4VOBnJXHv6WNVZYmtDFI8ceCLurFreWZFOeI1PbRVM0OmF2slcu
+        qpytOdohEKNcj5qeq+QAZrrr9NkfCS8zfryTeAsx1E7kk5wZFTq0drsCKVYj5RgT
+        59gUMBWCnUsvShKPZ4EcK+TQOM51WtaO2uZf0ro9ILLZDnnKOaOhd59omCX729Ad
+        QlAhaAAvB6kmrtUSLnttX3bTE6PafTG/ijoksqq7SeK1zg3siZFINe+L7u/HbE62
+        CGgyOb3yBu18xnmQra+J5qAXkhCUoECZZ1ot9pemwLJWsj+b4qy2lHbYda5dfsWM
+        wJ/KXhcyqjcyToe5KOxF5cOo0BLFpNwQ
+        -----END PRIVATE KEY-----
+
+  subgraph:
+    all:
+      client_authentication:
+        certificate_chain: *cert
+        key: *key
+    # if configuring for a specific subgraph:
+    subgraphs:
+      # subgraph name
+      products:
+        certificate_authorities: *cert
+        client_authentication:
+          certificate_chain: *cert
+          key: *key
\ No newline at end of file
diff --git a/apollo-router/src/router_factory.rs b/apollo-router/src/router_factory.rs
index 62dc5d888e..2ed2923d5c 100644
--- a/apollo-router/src/router_factory.rs
+++ b/apollo-router/src/router_factory.rs
@@ -290,32 +290,15 @@ pub(crate) async fn create_subgraph_services(
 
     let mut subgraph_services = IndexMap::new();
     for (name, _) in schema.subgraphs() {
-        let subgraph_root_store = configuration
-            .tls
-            .subgraph
-            .subgraphs
-            .get(name)
-            .as_ref()
-            .and_then(|subgraph| subgraph.create_certificate_store())
-            .transpose()?
-            .or_else(|| tls_root_store.clone());
-
         let subgraph_service = shaping.subgraph_service_internal(
             name,
-            SubgraphService::new(
+            SubgraphService::from_config(
                 name,
-                configuration
-                    .apq
-                    .subgraph
-                    .subgraphs
-                    .get(name)
-                    .map(|apq| apq.enabled)
-                    .unwrap_or(configuration.apq.subgraph.all.enabled),
-                subgraph_root_store,
+                configuration,
+                &tls_root_store,
                 shaping.enable_subgraph_http2(name),
                 subscription_plugin_conf.clone(),
-                configuration.notify.clone(),
-            ),
+            )?,
         );
         subgraph_services.insert(name.clone(), subgraph_service);
     }
@@ -360,14 +343,16 @@ impl YamlRouterFactory {
 }
 
 impl TlsSubgraph {
-    fn create_certificate_store(&self) -> Option> {
+    pub(crate) fn create_certificate_store(
+        &self,
+    ) -> Option> {
         self.certificate_authorities
             .as_deref()
             .map(create_certificate_store)
     }
 }
 
-fn create_certificate_store(
+pub(crate) fn create_certificate_store(
     certificate_authorities: &str,
 ) -> Result {
     let mut store = RootCertStore::empty();
diff --git a/apollo-router/src/services/subgraph_service.rs b/apollo-router/src/services/subgraph_service.rs
index 2d0044d488..9333bc16aa 100644
--- a/apollo-router/src/services/subgraph_service.rs
+++ b/apollo-router/src/services/subgraph_service.rs
@@ -37,6 +37,7 @@ use mediatype::names::JSON;
 use mediatype::MediaType;
 use mime::APPLICATION_JSON;
 use opentelemetry::global;
+use rustls::ClientConfig;
 use rustls::RootCertStore;
 use schemars::JsonSchema;
 use serde::Serialize;
@@ -75,6 +76,7 @@ use crate::query_planner::OperationKind;
 use crate::services::layers::apq;
 use crate::services::SubgraphRequest;
 use crate::services::SubgraphResponse;
+use crate::Configuration;
 use crate::Context;
 use crate::Notify;
 
@@ -151,28 +153,87 @@ pub(crate) struct SubgraphService {
 }
 
 impl SubgraphService {
+    pub(crate) fn from_config(
+        service: impl Into,
+        configuration: &Configuration,
+        tls_root_store: &Option,
+        enable_http2: bool,
+        subscription_config: Option,
+    ) -> Result {
+        let name: String = service.into();
+        let tls_cert_store = configuration
+            .tls
+            .subgraph
+            .subgraphs
+            .get(&name)
+            .as_ref()
+            .and_then(|subgraph| subgraph.create_certificate_store())
+            .transpose()?
+            .or_else(|| tls_root_store.clone());
+        let enable_apq = configuration
+            .apq
+            .subgraph
+            .subgraphs
+            .get(&name)
+            .map(|apq| apq.enabled)
+            .unwrap_or(configuration.apq.subgraph.all.enabled);
+        let client_cert_config = configuration
+            .tls
+            .subgraph
+            .subgraphs
+            .get(&name)
+            .as_ref()
+            .and_then(|tls| tls.client_authentication.as_ref())
+            .or(configuration
+                .tls
+                .subgraph
+                .all
+                .client_authentication
+                .as_ref());
+
+        let tls_builder = rustls::ClientConfig::builder().with_safe_defaults();
+        let tls_client_config = match (tls_cert_store, client_cert_config) {
+            (None, None) => tls_builder.with_native_roots().with_no_client_auth(),
+            (Some(store), None) => tls_builder
+                .with_root_certificates(store)
+                .with_no_client_auth(),
+            (None, Some(client_auth_config)) => {
+                tls_builder.with_native_roots().with_client_auth_cert(
+                    client_auth_config.certificate_chain.clone(),
+                    client_auth_config.key.clone(),
+                )?
+            }
+            (Some(store), Some(client_auth_config)) => tls_builder
+                .with_root_certificates(store)
+                .with_client_auth_cert(
+                    client_auth_config.certificate_chain.clone(),
+                    client_auth_config.key.clone(),
+                )?,
+        };
+
+        Ok(SubgraphService::new(
+            name,
+            enable_apq,
+            enable_http2,
+            subscription_config,
+            tls_client_config,
+            configuration.notify.clone(),
+        ))
+    }
+
     pub(crate) fn new(
         service: impl Into,
         enable_apq: bool,
-        tls_cert_store: Option,
         enable_http2: bool,
         subscription_config: Option,
+        tls_config: ClientConfig,
         notify: Notify,
     ) -> Self {
         let mut http_connector = HttpConnector::new();
         http_connector.set_nodelay(true);
         http_connector.set_keepalive(Some(std::time::Duration::from_secs(60)));
         http_connector.enforce_http(false);
-        let tls_config = match tls_cert_store {
-            None => rustls::ClientConfig::builder()
-                .with_safe_defaults()
-                .with_native_roots()
-                .with_no_client_auth(),
-            Some(store) => rustls::ClientConfig::builder()
-                .with_safe_defaults()
-                .with_root_certificates(store)
-                .with_no_client_auth(),
-        };
+
         let builder = hyper_rustls::HttpsConnectorBuilder::new()
             .with_tls_config(tls_config)
             .https_or_http()
@@ -1088,6 +1149,7 @@ where
 #[cfg(test)]
 mod tests {
     use std::convert::Infallible;
+    use std::io;
     use std::net::SocketAddr;
     use std::net::TcpListener;
     use std::str::FromStr;
@@ -1105,8 +1167,15 @@ mod tests {
     use http::header::HOST;
     use http::StatusCode;
     use http::Uri;
+    use http::Version;
+    use hyper::server::conn::AddrIncoming;
     use hyper::service::make_service_fn;
     use hyper::Body;
+    use hyper_rustls::TlsAcceptor;
+    use rustls::server::AllowAnyAuthenticatedClient;
+    use rustls::Certificate;
+    use rustls::PrivateKey;
+    use rustls::ServerConfig;
     use serde_json_bytes::ByteString;
     use serde_json_bytes::Value;
     use tower::service_fn;
@@ -1115,6 +1184,10 @@ mod tests {
     use SubgraphRequest;
 
     use super::*;
+    use crate::configuration::load_certs;
+    use crate::configuration::load_key;
+    use crate::configuration::TlsClientAuth;
+    use crate::configuration::TlsSubgraph;
     use crate::graphql::Error;
     use crate::graphql::Request;
     use crate::graphql::Response;
@@ -1770,9 +1843,12 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "testbis",
             true,
-            None,
             false,
             subscription_config().into(),
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
             Notify::builder().build(),
         );
         let (tx, _rx) = mpsc::channel(2);
@@ -1819,8 +1895,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_subgraph_application_graphql_response(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let response = subgraph_service
@@ -1853,8 +1938,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_subgraph_application_json_response(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let response = subgraph_service
@@ -1887,8 +1981,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_subgraph_ok_status_invalid_response(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let response = subgraph_service
@@ -1926,8 +2029,17 @@ mod tests {
         tokio::task::spawn(
             emulate_subgraph_invalid_response_invalid_status_application_json(listener),
         );
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let response = subgraph_service
@@ -1969,8 +2081,17 @@ mod tests {
         tokio::task::spawn(
             emulate_subgraph_invalid_response_invalid_status_application_graphql(listener),
         );
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let response = subgraph_service
@@ -2013,9 +2134,12 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            None,
             false,
             subscription_config().into(),
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
             Notify::builder().build(),
         );
         let (tx, mut rx) = mpsc::channel(2);
@@ -2073,9 +2197,12 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            None,
             false,
             subscription_config().into(),
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
             Notify::builder().build(),
         );
         let (tx, _rx) = mpsc::channel(2);
@@ -2122,8 +2249,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_subgraph_bad_request(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let response = subgraph_service
@@ -2164,8 +2300,17 @@ mod tests {
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_subgraph_bad_response_format(listener));
 
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let response = subgraph_service
@@ -2201,8 +2346,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_subgraph_compressed_response(listener));
-        let subgraph_service =
-            SubgraphService::new("test", false, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            false,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let resp = subgraph_service
@@ -2242,8 +2396,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_subgraph_unauthorized(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let response = subgraph_service
@@ -2279,8 +2442,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_persisted_query_not_supported_message(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         assert!(subgraph_service.clone().apq.as_ref().load(Relaxed));
 
@@ -2325,8 +2497,17 @@ mod tests {
         tokio::task::spawn(emulate_persisted_query_not_supported_extension_code(
             listener,
         ));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         assert!(subgraph_service.clone().apq.as_ref().load(Relaxed));
 
@@ -2369,8 +2550,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_persisted_query_not_found_message(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let resp = subgraph_service
@@ -2410,8 +2600,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_persisted_query_not_found_extension_code(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let resp = subgraph_service
@@ -2451,8 +2650,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_expected_apq_enabled_configuration(listener));
-        let subgraph_service =
-            SubgraphService::new("test", true, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let resp = subgraph_service
@@ -2492,8 +2700,17 @@ mod tests {
         let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
         let socket_addr = listener.local_addr().unwrap();
         tokio::task::spawn(emulate_expected_apq_disabled_configuration(listener));
-        let subgraph_service =
-            SubgraphService::new("test", false, None, true, None, Notify::default());
+        let subgraph_service = SubgraphService::new(
+            "test",
+            false,
+            true,
+            None,
+            ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
 
         let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
         let resp = subgraph_service
@@ -2527,4 +2744,245 @@ mod tests {
 
         assert_eq!(resp.response.body(), &expected_resp);
     }
+
+    async fn tls_server(
+        listener: tokio::net::TcpListener,
+        certificates: Vec,
+        key: PrivateKey,
+        body: &'static str,
+    ) {
+        let acceptor = TlsAcceptor::builder()
+            .with_single_cert(certificates, key)
+            .unwrap()
+            .with_all_versions_alpn()
+            .with_incoming(AddrIncoming::from_listener(listener).unwrap());
+        let service = make_service_fn(|_| async {
+            Ok::<_, io::Error>(service_fn(|_req| async {
+                Ok::<_, io::Error>(
+                    http::Response::builder()
+                        .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                        .status(StatusCode::OK)
+                        .version(Version::HTTP_11)
+                        .body::(body.into())
+                        .unwrap(),
+                )
+            }))
+        });
+        let server = Server::builder(acceptor).serve(service);
+        server.await.unwrap()
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn tls_self_signed() {
+        let certificate_pem = include_str!("./testdata/server_self_signed.crt");
+        let key_pem = include_str!("./testdata/server.key");
+
+        let certificates = load_certs(certificate_pem).unwrap();
+        let key = load_key(key_pem).unwrap();
+
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let socket_addr = listener.local_addr().unwrap();
+        tokio::task::spawn(tls_server(listener, certificates, key, r#"{"data": null}"#));
+
+        // we cannot parse a configuration from text, because certificates are generally
+        // added by file expansion and we don't have access to that here, and inserting
+        // the PEM data directly generates parsing issues due to end of line characters
+        let mut config = Configuration::default();
+        config.tls.subgraph.subgraphs.insert(
+            "test".to_string(),
+            TlsSubgraph {
+                certificate_authorities: Some(certificate_pem.into()),
+                client_authentication: None,
+            },
+        );
+        let subgraph_service =
+            SubgraphService::from_config("test", &config, &None, false, None).unwrap();
+
+        let url = Uri::from_str(&format!("https://localhost:{}", socket_addr.port())).unwrap();
+        let response = subgraph_service
+            .oneshot(SubgraphRequest {
+                supergraph_request: Arc::new(
+                    http::Request::builder()
+                        .header(HOST, "host")
+                        .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                        .body(Request::builder().query("query").build())
+                        .expect("expecting valid request"),
+                ),
+                subgraph_request: http::Request::builder()
+                    .header(HOST, "rhost")
+                    .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                    .uri(url)
+                    .body(Request::builder().query("query").build())
+                    .expect("expecting valid request"),
+                operation_kind: OperationKind::Query,
+                context: Context::new(),
+                subscription_stream: None,
+                connection_closed_signal: None,
+            })
+            .await
+            .unwrap();
+        assert_eq!(response.response.body().data, Some(Value::Null));
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn tls_custom_root() {
+        let certificate_pem = include_str!("./testdata/server.crt");
+        let ca_pem = include_str!("./testdata/CA/ca.crt");
+        let key_pem = include_str!("./testdata/server.key");
+
+        let mut certificates = load_certs(certificate_pem).unwrap();
+        certificates.extend(load_certs(ca_pem).unwrap());
+        let key = load_key(key_pem).unwrap();
+
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let socket_addr = listener.local_addr().unwrap();
+        tokio::task::spawn(tls_server(listener, certificates, key, r#"{"data": null}"#));
+
+        // we cannot parse a configuration from text, because certificates are generally
+        // added by file expansion and we don't have access to that here, and inserting
+        // the PEM data directly generates parsing issues due to end of line characters
+        let mut config = Configuration::default();
+        config.tls.subgraph.subgraphs.insert(
+            "test".to_string(),
+            TlsSubgraph {
+                certificate_authorities: Some(ca_pem.into()),
+                client_authentication: None,
+            },
+        );
+        let subgraph_service =
+            SubgraphService::from_config("test", &config, &None, false, None).unwrap();
+
+        let url = Uri::from_str(&format!("https://localhost:{}", socket_addr.port())).unwrap();
+        let response = subgraph_service
+            .oneshot(SubgraphRequest {
+                supergraph_request: Arc::new(
+                    http::Request::builder()
+                        .header(HOST, "host")
+                        .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                        .body(Request::builder().query("query").build())
+                        .expect("expecting valid request"),
+                ),
+                subgraph_request: http::Request::builder()
+                    .header(HOST, "rhost")
+                    .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                    .uri(url)
+                    .body(Request::builder().query("query").build())
+                    .expect("expecting valid request"),
+                operation_kind: OperationKind::Query,
+                context: Context::new(),
+                subscription_stream: None,
+                connection_closed_signal: None,
+            })
+            .await
+            .unwrap();
+        assert_eq!(response.response.body().data, Some(Value::Null));
+    }
+
+    async fn tls_server_with_client_auth(
+        listener: tokio::net::TcpListener,
+        certificates: Vec,
+        key: PrivateKey,
+        client_root: Certificate,
+        body: &'static str,
+    ) {
+        let mut client_auth_roots = RootCertStore::empty();
+        client_auth_roots.add(&client_root).unwrap();
+
+        let client_auth = AllowAnyAuthenticatedClient::new(client_auth_roots).boxed();
+
+        let acceptor = TlsAcceptor::builder()
+            .with_tls_config(
+                ServerConfig::builder()
+                    .with_safe_defaults()
+                    .with_client_cert_verifier(client_auth)
+                    .with_single_cert(certificates, key)
+                    .unwrap(),
+            )
+            .with_all_versions_alpn()
+            .with_incoming(AddrIncoming::from_listener(listener).unwrap());
+        let service = make_service_fn(|_| async {
+            Ok::<_, io::Error>(service_fn(|_req| async {
+                Ok::<_, io::Error>(
+                    http::Response::builder()
+                        .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                        .status(StatusCode::OK)
+                        .version(Version::HTTP_11)
+                        .body::(body.into())
+                        .unwrap(),
+                )
+            }))
+        });
+        let server = Server::builder(acceptor).serve(service);
+        server.await.unwrap()
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn tls_client_auth() {
+        let server_certificate_pem = include_str!("./testdata/server.crt");
+        let ca_pem = include_str!("./testdata/CA/ca.crt");
+        let server_key_pem = include_str!("./testdata/server.key");
+
+        let mut server_certificates = load_certs(server_certificate_pem).unwrap();
+        let ca_certificate = load_certs(ca_pem).unwrap().remove(0);
+        server_certificates.push(ca_certificate.clone());
+        let key = load_key(server_key_pem).unwrap();
+
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let socket_addr = listener.local_addr().unwrap();
+        tokio::task::spawn(tls_server_with_client_auth(
+            listener,
+            server_certificates,
+            key,
+            ca_certificate,
+            r#"{"data": null}"#,
+        ));
+
+        let client_certificate_pem = include_str!("./testdata/client.crt");
+        let client_key_pem = include_str!("./testdata/client.key");
+
+        let client_certificates = load_certs(client_certificate_pem).unwrap();
+        let client_key = load_key(client_key_pem).unwrap();
+
+        // we cannot parse a configuration from text, because certificates are generally
+        // added by file expansion and we don't have access to that here, and inserting
+        // the PEM data directly generates parsing issues due to end of line characters
+        let mut config = Configuration::default();
+        config.tls.subgraph.subgraphs.insert(
+            "test".to_string(),
+            TlsSubgraph {
+                certificate_authorities: Some(ca_pem.into()),
+                client_authentication: Some(TlsClientAuth {
+                    certificate_chain: client_certificates,
+                    key: client_key,
+                }),
+            },
+        );
+        let subgraph_service =
+            SubgraphService::from_config("test", &config, &None, false, None).unwrap();
+
+        let url = Uri::from_str(&format!("https://localhost:{}", socket_addr.port())).unwrap();
+        let response = subgraph_service
+            .oneshot(SubgraphRequest {
+                supergraph_request: Arc::new(
+                    http::Request::builder()
+                        .header(HOST, "host")
+                        .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                        .body(Request::builder().query("query").build())
+                        .expect("expecting valid request"),
+                ),
+                subgraph_request: http::Request::builder()
+                    .header(HOST, "rhost")
+                    .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                    .uri(url)
+                    .body(Request::builder().query("query").build())
+                    .expect("expecting valid request"),
+                operation_kind: OperationKind::Query,
+                context: Context::new(),
+                subscription_stream: None,
+                connection_closed_signal: None,
+            })
+            .await
+            .unwrap();
+        assert_eq!(response.response.body().data, Some(Value::Null));
+    }
 }
diff --git a/apollo-router/src/services/testdata/CA/ca.crt b/apollo-router/src/services/testdata/CA/ca.crt
new file mode 100644
index 0000000000..6b12c0f9ef
--- /dev/null
+++ b/apollo-router/src/services/testdata/CA/ca.crt
@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFYTCCA0mgAwIBAgIUFhEXodpzYVGQ/3LlWZnw+ogPBTcwDQYJKoZIhvcNAQEL
+BQAwPzELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFwaFFMMRcwFQYD
+VQQDDA5BcG9sbG8gVGVzdCBDQTAgFw0yMzA5MDgxNDA2NDhaGA8yMDUxMDEyNDE0
+MDY0OFowPzELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFwaFFMMRcw
+FQYDVQQDDA5BcG9sbG8gVGVzdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
+AgoCggIBALu8ZVZoMc89GKXtnUKwJv6A4Wm/ZMeFJLhr2TuD6ahAGq5LmNNxdTt6
+SA1OMFSgZKjXrip5tIZPRHtxajNJLziIssegf7yUmhmnwMSsQo4ZL7WfmbXuvgKD
+/z3NthFwk93OYHMNlJ7hunhzW9mgka/FyqjsnyhX4HTgiWalX/ar2TQ1z1w8T9gz
+WgRqWTJhWJQO9lAMw43BTMURzo+f/1y/7ooTHkGwgPtWxgquSONsjOAhtfFhoCY9
+XB5xsKSOav0cNmwF2+16j1Z5znjIsBLBbno8HODBlA6abiE1XMmo2lsRtbHvSdHP
+bElZd298tppSbDTIgWDDcL7hchpHGfBFho167gKIdEcioUzCuyhGH4XYHJco8gsG
+cZFX2MfeuUNp5v/qjndBUqRwMqrqvU7iQTygJRuMlU0+ULW0niFvQCiNlACzQfOV
+0zh+YZnPl0cP1UfElTcyWGN0x/8HLR06qyfLIragh93kpC1wX00YUpVvtGkZc3Y8
+XYgwR5Dw+XsyvrCzba8d0w/vHKEPF4uFMYSNN1viXN3AyswJsGNnwkVgwpg37W25
+kHqHBgqLxCup6+YGRzEhrKIhdHmzJss2RafCrpjcmA/Meuaup+85lbdTcdfcNcL4
+d7zkvIeoFU/NucUguLAztS0X05rOcNsq/4uZzslNhj1UcdkEbuHtAgMBAAGjUzBR
+MB0GA1UdDgQWBBSz6p5WY8/jvOA/XR/YxIk3Y6DzFjAfBgNVHSMEGDAWgBSz6p5W
+Y8/jvOA/XR/YxIk3Y6DzFjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA
+A4ICAQBkegjS3E4TOqchNINpRB+ChnLCUxVP/yQCxPtouF3MR91YNjXb6dIZChWS
+Hrt2nB1LBtJDJFK6LEJXgz6AJniKzBotG8mPRmmNPQrSSXKnaCDRyWZ7+0My7x5Q
+XvEziLidJA9itb6+Gys1PuUHQdWAC22FVNjlKAodPgtQaQrse5DVZWqK55OLTIi4
+p2SbrfiYe9Zs+ZGwlx2UAPilV3l/2cEJPrP0pNGePeLzvgfzmBFgyah75Y7tDxJ6
+1qIc2sifz0csN4uKbqGn7MG9Y6hw6ML5jjrHVdqizeM3GQzHADAzJfYZYn5gB1Hk
+Z6zfNRiN5CrmU9ECIh+EgI6N1yDni6bzrF2mHszd3ShvoztvEAFRkeL43owaLC3y
+Z/2asGC+PO3iDr9P13S8n/dnrFjAzi9ociijwwPqwm1YRjmO5yjzXWoSoQ1HGwBw
+E9IbhqGUBjZWGqVPkfsy3Ox0HRpXT1PK9cl560iqKy0/bTVKQkVNk2LEbMCOTEYQ
+453POBgkXY+W1uktVJOv0Yk8+9Yc2dZgqhJc6Xxz9bRXmJQZlE1pAXc3VNCic/W3
+8VLj5+R82vZk3+2QFTy3ZSIMZAlO8s22kM/tqXB+nAVTsrqBiHXRvph6S/deO6AF
+o1O4Fwifrf2G6ln9bs+zdMlX1xbGgwxubLwp4I96yvtaSIBTuA==
+-----END CERTIFICATE-----
diff --git a/apollo-router/src/services/testdata/CA/ca.key b/apollo-router/src/services/testdata/CA/ca.key
new file mode 100644
index 0000000000..58742c835d
--- /dev/null
+++ b/apollo-router/src/services/testdata/CA/ca.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC7vGVWaDHPPRil
+7Z1CsCb+gOFpv2THhSS4a9k7g+moQBquS5jTcXU7ekgNTjBUoGSo164qebSGT0R7
+cWozSS84iLLHoH+8lJoZp8DErEKOGS+1n5m17r4Cg/89zbYRcJPdzmBzDZSe4bp4
+c1vZoJGvxcqo7J8oV+B04IlmpV/2q9k0Nc9cPE/YM1oEalkyYViUDvZQDMONwUzF
+Ec6Pn/9cv+6KEx5BsID7VsYKrkjjbIzgIbXxYaAmPVwecbCkjmr9HDZsBdvteo9W
+ec54yLASwW56PBzgwZQOmm4hNVzJqNpbEbWx70nRz2xJWXdvfLaaUmw0yIFgw3C+
+4XIaRxnwRYaNeu4CiHRHIqFMwrsoRh+F2ByXKPILBnGRV9jH3rlDaeb/6o53QVKk
+cDKq6r1O4kE8oCUbjJVNPlC1tJ4hb0AojZQAs0HzldM4fmGZz5dHD9VHxJU3Mlhj
+dMf/By0dOqsnyyK2oIfd5KQtcF9NGFKVb7RpGXN2PF2IMEeQ8Pl7Mr6ws22vHdMP
+7xyhDxeLhTGEjTdb4lzdwMrMCbBjZ8JFYMKYN+1tuZB6hwYKi8QrqevmBkcxIayi
+IXR5sybLNkWnwq6Y3JgPzHrmrqfvOZW3U3HX3DXC+He85LyHqBVPzbnFILiwM7Ut
+F9OaznDbKv+Lmc7JTYY9VHHZBG7h7QIDAQABAoICABKsCv80V0nnp3kvYC4HE6XJ
+/0FziAxBHZUbIuzEsrtpfs3Zln5nQTP0dlCed3efSUuywe+XEcGvBh5p+hhWPdRX
+YkKxH6tLs2T8VpOpF2iHMEPV9yWNdIgPBVaqDGMBKa/MrDKdRewkgtWikZ7cVckf
++Mdi23SZgulVAM19AsWns296L6Danc1g1CHuyubUIEl/kHeHQXbqpB5QplUKIo2g
+yFKi0qZRbg6xg3qB+Z1nCT7sk2pJ3bIX9b7Qa3N+d70rbm1Biu18fF+rN0IzobGH
+3BtbubqP/lASg4QZiNxEx0z5/rKPHG45/WjxNUwZIRdJvdnlHFsZrS/K//yFmFpt
+uJLEVqYS7WPbU5Dm3swSQGeEzkD0LLEcE5ELjkjy3DmpVRbRYG42KvBvgoYudSYi
+vTicgqDQOowlZ1Cr1+bZxXNTn7shb8P6j2AEYDM3kDpFGYu4WlCBbCmyOnYuRvqr
+IGlmykd7s32+jQlSgSnHtjPoS/QbvzZhITqNqAxfxvPsWmqk8yXVMBjM7QGUh3Ny
+WwyFL5HUy47FewYT0A05uIvE43acqAonVub8qk8NZPP5TTa6jHUeCIuLyJ5m3HJ7
+XaQTxamGy29tZIbgk0xmPdCrDloivGpPeYGT7phsytCPclxnWaiK/y5dBbB8ut7E
+Ak+ejXdmdZZvIb02J6dRAoIBAQD4bQkcHEaGHuh2MdMT8wmr72ivIg3uzJSTR8hL
+mpWWXAHqFSkyWw9jMBLCfpxGe/qZHz1/rHSzAGsS9tieGSHKN6zL0SMhaSAw4KoX
+oneVPOUkDJ08EsYtELXhaT6t4cDdQflJaS5WF+xo2rVnG8DLtN7lsbQffFj8Pp+N
+agG8wRbTnTaqPBoptwJraDGXRAorIY5+la4SrPebnO+oR7S126CAxGBRGhBL5p/Z
+yz3uQFKsfoR1LTSu8WPhvPW3RfRgPR7jb5wiLwNBgWOd0fF9CR39FBYNoP2ExaVk
+Md4u0oXQX0s0PJkIDv1fDBmScHRvwFyD5gMGmht9bJfGYpYxAoIBAQDBday/dQVc
+XIu+AQHb9YNeWDjXGb+7oM1sE7zjOkJNB/YAfgL2oyNFbjKT5FFelE/wwGLV1zOs
+eHu0We3uMRTTjTUM4YTaKjmkYZx7Li2i+OZ6jadYn3iFxOtUOvxeyVhf17zjUHt/
+d+2N8LqEJ2MQ7fcqQV91X1BfSl5RA4RP7OEYS8P0mjog1gRr7AcsdR7C2zX1BWpc
+5wIT3gEHpvNL55ZG/Z6R5hOqhFMW06gDcQpWhgjOaUjJvMMNT8PdShxFSG0q3AEP
+Uq7wcRT5y2rjX5COBun4qV5uCZzUxLwYGLjgG2DKKpSvamWPFmgXlwxME+iLXZuU
+NHhLNPWuxUx9AoIBAQCRneAcIRrZnR5nwbbXV7uQk8nJ9m0yT6q7VfU7E8ahFxds
+LbEzOhsf6drlNK/HxP5UaPJ0dxMEpbcdq6G3skWTMeVQpysvd16OJ/Kr3ei4irRJ
+2DcGMn8FhWkHUtiRteWGDBjihgvKHUXPBHwWjzcIcOHCw9IwLoQ+fzqo+r0/9fK6
+YcncKVSwpPx/FtkSsKhBNe8/ld8uRULybSAxhK6RdsZzFCjdASkSRIHoePC5Sezr
+/VEJGlFqRkZk2I0zO4cOLmV/rTlY9DO1XvGUXa5uuzxfzzlKDdbsz6mtvGlfCrmz
+NVhwqyCMj7TG1OHca5xksN3J1rZpVUXN1Cx29ezRAoIBAQCXi6DZEcgzQ+PL4Wod
+bkOMQju0dF03k42XnhaXQWnygcv5Mr0Li1vz5ZEFPRNorXG5vFsv/DZkYrU04EV8
+E0KnPbRS9M5nQDCLTlOQQupJAgU594Eh8TouP8XA47wSXFM4MP2sa71yavXR/MN/
+BoOaHW9C0JcAkLubCdy8Ra7+FSd3KtLD55SlaEFqcShlcRG5vpKT77wULnn4Ki5Q
+Wj24NiAIWGBBFiRNrP6RoRe6KnDYuoJnfx4fuhhpcxSRYp6mv0eVDWZfJHwZqduu
+meSPXKsWNuBNhA+Oz0Tlf3+AuM0J8Shu9dAnYYdRAVJlOFPPvMCre5aGEFwzPbc/
+QxrZAoIBAGi+6m2cgUwxVguTpdJM9FeE3ret3tOhbxyyec8XOjI5LXzoKAygIjjF
+snoDW99fnYZy7JNl0nnXKTGsRrMQg//hDJJ8JhyVoTnd91DEJ3gesRYZhqZPXkC4
+hURR0KSHPPcxCceaLpEU5pQk6ehjKO/CdItfOzrAbpc4VjSbPN9Fw8UPdYWJhN7k
+IzjRhfChGjdyZ0ejRw2ZvjJITyejJJxCDRsryKa3Xr2ot6e3Xnq9DBTTOqzJmlxM
+YVCHKksAn+o1dXElm6aHdkaEWmGEkYD/NF6CZ31cA9cZjHFsd3B733DlrNaGDwS2
+WGzJrJ309AV6WL7hOxRUJFHSmPh/zN0=
+-----END PRIVATE KEY-----
diff --git a/apollo-router/src/services/testdata/client.crt b/apollo-router/src/services/testdata/client.crt
new file mode 100644
index 0000000000..289aa64b4e
--- /dev/null
+++ b/apollo-router/src/services/testdata/client.crt
@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFzDCCA7SgAwIBAgIUa67eWr71k4kvpbd7NzXgBDepDzswDQYJKoZIhvcNAQEL
+BQAwPzELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFwaFFMMRcwFQYD
+VQQDDA5BcG9sbG8gVGVzdCBDQTAgFw0yMzA5MDgxNDA5NTNaGA8yMDUxMDEyNDE0
+MDk1M1owNzELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFwaFFMMQ8w
+DQYDVQQDDAZyb3V0ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDZ
+FAn0/6v8Q5K8JE4UhiTAL3utNBO33tKWVxCXIy1vnM9cfJ6hrpM8/qCFAc8HMs0S
+HfMuEWRSO9k/7C10L1I6n/O7jiFOr1hbgiQO9twXP1Ic+R1HnZrXv9A0Ql2zXw0I
+7SiZc7sgUBLsBmBOvCKhpJnlPuE0p5WTutpUSk6jp8SFqEjYVBxPWO+kSSxT3idU
+ad9Tw6jm2WS40NGRqrHaUlnpYxDqOf2yKHHH/6PIVAdXnPPH4d9rnS/f9OphG07W
+vArrwb9fvCbXF7vNSIGXa7jfMK2xJUvf+KWj6g+0QtXzf1H+5EMC5jVcfWz0KUJ1
+YFJdDWU7DzmwFM5Ow7hrTtzv5HQoh8aEy62xZeiWFuWPCgttPx1WSrVsoe0WVmaE
+YY3enwkts2aM36NprUH2MaMDXwyXvroPaP/qSXgPk2Oa2SR8xam1jJuVtBt7JNSG
+PcZdi8niySKa2tHke2ZhdrwIzASZQZUudm6i8b+SgTsqcfuFN7EeH7kKNCceyuwX
+R6JTJPSKVor91DlU48ldwlmxpKB4+gZIwc0WGzzWkrRjD6S3E9NKIhpE+xT+KVSU
+u7PI+eIzDb9njZSGb0yIqr+ijQPJ18lj7ZqyF3CzW0+TT/iDNIZa6J0fBH95A1qY
+I2ini9XKBPImVTg+f3LzCfQb1UWgwzAYxgk6G9S4oQIDAQABo4HFMIHCMAkGA1Ud
+EwQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NM
+IEdlbmVyYXRlZCBDbGllbnQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFO/R+Ko2820I
+7K3OGCgXTrd7gUzbMB8GA1UdIwQYMBaAFLPqnlZjz+O84D9dH9jEiTdjoPMWMA4G
+A1UdDwEB/wQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJ
+KoZIhvcNAQELBQADggIBAJaVSLu9kMHL91UOXCeN45NRSvw2mjrsx7ZeYYhYJX0v
+iEatXAvyMt9FPq9lE6oGI4SoYQnYlwv/HjEXbOXpXdQN2ow4oBsEWXNq7PW5x+nv
+5OK7McZnOXMpSq27jUFC/0UqRh+KcxaGpZtiRg+GyV58ITcC7XBEzcqv/KIjniGE
+st5U2jkOxvldogxGSwLCrEBiLAvAAi9i4dxT3Or6E/xX9WbtYhgFuzjmEEzZYFmU
+wrTTvCr7mBSbldDzugVCMi5cP2sXWcKxO8hMNYDZySYhAZie/lljfHTpPBqfufNL
+dfhim2I/b+r8hgA2ZhrQSXUy5yAitaKCVo3p6J+1WC76dyNiNbpWhQtPSSbSXgfO
+yr088/xLkm0DuL0CxCA4cx2Fg5PzkCSIP8RHYig3hYDFGU3zrA4sC3Gjsy/ia0wU
+c7zglv8nnS2wcsWHzAeVwz5PkLwrGzUvxXFAequGg/XK7YGqrzvnFsiURDLrMszi
+DiIfJqLFFwzvI2FgI6GM6pMH+x0ahhngipeSXuUoM89sz7xT9VFmNpQcDC2I5LwC
+19sVTxwDkXvLuZU326a/Md97fbzbLhxzr0EPlWZLhdosUl15BBtU2Il88UGeDBWN
+UHSmawen6WtZW89o5oH3YSD6tZtWmyKX5Kdq7u8Ud5WIdkGRzJzLZ2Tbs6X4NYI1
+-----END CERTIFICATE-----
diff --git a/apollo-router/src/services/testdata/client.csr b/apollo-router/src/services/testdata/client.csr
new file mode 100644
index 0000000000..b452f5bb2b
--- /dev/null
+++ b/apollo-router/src/services/testdata/client.csr
@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIEfDCCAmQCAQAwNzELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFw
+aFFMMQ8wDQYDVQQDDAZyb3V0ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
+AoICAQDZFAn0/6v8Q5K8JE4UhiTAL3utNBO33tKWVxCXIy1vnM9cfJ6hrpM8/qCF
+Ac8HMs0SHfMuEWRSO9k/7C10L1I6n/O7jiFOr1hbgiQO9twXP1Ic+R1HnZrXv9A0
+Ql2zXw0I7SiZc7sgUBLsBmBOvCKhpJnlPuE0p5WTutpUSk6jp8SFqEjYVBxPWO+k
+SSxT3idUad9Tw6jm2WS40NGRqrHaUlnpYxDqOf2yKHHH/6PIVAdXnPPH4d9rnS/f
+9OphG07WvArrwb9fvCbXF7vNSIGXa7jfMK2xJUvf+KWj6g+0QtXzf1H+5EMC5jVc
+fWz0KUJ1YFJdDWU7DzmwFM5Ow7hrTtzv5HQoh8aEy62xZeiWFuWPCgttPx1WSrVs
+oe0WVmaEYY3enwkts2aM36NprUH2MaMDXwyXvroPaP/qSXgPk2Oa2SR8xam1jJuV
+tBt7JNSGPcZdi8niySKa2tHke2ZhdrwIzASZQZUudm6i8b+SgTsqcfuFN7EeH7kK
+NCceyuwXR6JTJPSKVor91DlU48ldwlmxpKB4+gZIwc0WGzzWkrRjD6S3E9NKIhpE
++xT+KVSUu7PI+eIzDb9njZSGb0yIqr+ijQPJ18lj7ZqyF3CzW0+TT/iDNIZa6J0f
+BH95A1qYI2ini9XKBPImVTg+f3LzCfQb1UWgwzAYxgk6G9S4oQIDAQABoAAwDQYJ
+KoZIhvcNAQELBQADggIBAI2SS4y1+rngn3Za3VlcPfd/+g+L/NOwVC19To5vtx7h
+11ZmQMh6kWC0e+GbiBbK4+KZqFo120u7e+PBdiG+nM+qlCAuGi3f41DHHASHqf3r
++bw7/u3N4ZuzadVVNX4zamLlZ1NpWUqwPXi59MOdjYhAtz92nbNR9swKGzxVg4rk
+U0g1FB0hXY7RGbkdURFGPDKCUHw4wAWK4KcTFUXmAHAX0fdGfBJe1ebFCS7bNmKr
+C9NrwKV96swzWINsnpqk8i1M37KN7YgXyJPE7G4Tj+8hn+dc6E7Y60LJDwkTnx4v
+yQycWnY6LP8MMT9quFPuaaadzAfQu4lRKc05TtxDIZf+2Pipj+h7TDJY2nbcDsY0
+b45gh7T0IrNkDdszfCSZbjYmVDg2Nxuy9i6hKMGngVllL+wJ8rjmXC5HRsCvgSyY
+804z8OuK72xDD5x4zQEMJ40Tr+1wyqT5jWiO4iyj0faavdP9TrNZaLYeF/nfwNt5
+pr1A9TqcCLIc1Zid/i2/++/2jPT+0Fc7mpmSvxfc+CGv74d1Cb5pp/U/8uvukTQV
+X/4WkWo/+KGg/WYgYBla7ysjursankZ0WYX5MCNbCJ6xSq9z6Y6U+ZK+SsC49S1B
+G/H6UUnPGkA5k7EFJMFVAamBHVlAxOLzvx/poawxeIqEKJYZ2OewhKJNhw7tKrpp
+-----END CERTIFICATE REQUEST-----
diff --git a/apollo-router/src/services/testdata/client.ext b/apollo-router/src/services/testdata/client.ext
new file mode 100644
index 0000000000..bc73c04527
--- /dev/null
+++ b/apollo-router/src/services/testdata/client.ext
@@ -0,0 +1,7 @@
+basicConstraints = CA:FALSE
+nsCertType = client, email
+nsComment = "OpenSSL Generated Client Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth, emailProtection
diff --git a/apollo-router/src/services/testdata/client.key b/apollo-router/src/services/testdata/client.key
new file mode 100644
index 0000000000..bc6468071b
--- /dev/null
+++ b/apollo-router/src/services/testdata/client.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDZFAn0/6v8Q5K8
+JE4UhiTAL3utNBO33tKWVxCXIy1vnM9cfJ6hrpM8/qCFAc8HMs0SHfMuEWRSO9k/
+7C10L1I6n/O7jiFOr1hbgiQO9twXP1Ic+R1HnZrXv9A0Ql2zXw0I7SiZc7sgUBLs
+BmBOvCKhpJnlPuE0p5WTutpUSk6jp8SFqEjYVBxPWO+kSSxT3idUad9Tw6jm2WS4
+0NGRqrHaUlnpYxDqOf2yKHHH/6PIVAdXnPPH4d9rnS/f9OphG07WvArrwb9fvCbX
+F7vNSIGXa7jfMK2xJUvf+KWj6g+0QtXzf1H+5EMC5jVcfWz0KUJ1YFJdDWU7Dzmw
+FM5Ow7hrTtzv5HQoh8aEy62xZeiWFuWPCgttPx1WSrVsoe0WVmaEYY3enwkts2aM
+36NprUH2MaMDXwyXvroPaP/qSXgPk2Oa2SR8xam1jJuVtBt7JNSGPcZdi8niySKa
+2tHke2ZhdrwIzASZQZUudm6i8b+SgTsqcfuFN7EeH7kKNCceyuwXR6JTJPSKVor9
+1DlU48ldwlmxpKB4+gZIwc0WGzzWkrRjD6S3E9NKIhpE+xT+KVSUu7PI+eIzDb9n
+jZSGb0yIqr+ijQPJ18lj7ZqyF3CzW0+TT/iDNIZa6J0fBH95A1qYI2ini9XKBPIm
+VTg+f3LzCfQb1UWgwzAYxgk6G9S4oQIDAQABAoIB/xyByfnA+Zz1lfd1hqtuhYHK
+uofsWgqNDUQ6lGGReEE7OCAB9yY6y5GSMd8Sdbv3d+bBK1MJbA5zvZWF0BJJ6v23
+5t7igCbZ8AiHkD5G1nBXG5/VnXq7v76r4EB8o3SpLtUIdlejuX2gBlwbs9oJhG+M
+Mu/5nSmFBUEscJXqv2XOfYhWQsd/hDHUKn3jyuZyOuGWMoKVinPszviJgHvyQDKJ
+55s2knbjI39mwEpunKhaIJZPQW3zKgJZcHtAqbOtTZUouWZscyCA9oWYoEw9EVRv
+pdkcucwua0bMwgCpZDApkF8054rjMjAKe0d1UTGSZlMNUZBwVrLpNPIvfHuFFSdq
+5GaxW8uvjNN4ILwbEr5DVGaqUfuwfmDWC3o0QmJJ6rSH0O6sGPtoT3GkfWPhG3B7
+ywhqDrLiMf8g1TwCFD+O04pQMQlX8fwX65PFV/i+kvkEZ7XirBTkDRo7UBMzyXZb
+rxJ/LApUIPwEz5PxvX+47xlHGgBRXnSOeCjXxLVZoiNHPXP1MFupE8dpvFEbW5t7
+z1MbT4MutSeV1ez3uEFbubUwBSVjjoSoumBwodYuLTmMSuAabzVOuNPX5VE0GM/Y
+2gygAOI6LYzbyLlQzKk/e7XoPiWoC9MYW5SdHjW8Qup1SuL9NgKI5jLmGZAGKWUR
+VeObPdMUD7LYJkLOvsECggEBAO0iKFOT0u5T7ERA6htFKNNcN6E3g8rg6t5xQPFz
+Kz9hJbLcTP2doZBePu1b4OLDDQ5iMHD09C8E6f2NIGj+SXsyTTrVk3X3pyx9Cdik
+tdjJ4jm4DKdFg3Zv2tVKm9jDFiWIfz1WX7Hm7Z3N12knXvk6yA1kRjmF4gVXvTiv
+iPc9mbA6ggwRFib8URQg6JRCPga7ez9Ek/I2o192Tb87SY8ln9n9u7yCh2SAVDL9
+ocO8CaDUkoj5EROtS46LyN74eDYlFdt2U6pOJ2eprRmr0dnd9NJxcmMGD90U2Kki
+4hx0BvgUzyQcZ5QRMvCzUAs4iiEBXO5Nv8TZE9aGkraWFMECggEBAOpZZ+BgfmG/
+y9RBxY9la/DiYA/Pf1gkfRdH0+/H0qRpmSVqy9ROh9zYCQK6RNN2oT+Ps8vBxyJK
+QvxVO4AkDwKKgVXqZeEWNqEwJBQJCBzQfqybXA24pFQE5fDB0a6WbZLQByiXMWr7
+xu6m0Rw7KXz+v31iVyouS3jCLAC6QLu10QcVVs0WivzK9bxqPsY53kNQ2/ysOl6D
+vTv1UzWJ6zlLJ2+AvzX6zWBNF26xQ0SDU0gMY9RBtcP3STw0HLTqJhuxLvhxvnmK
+cnQSLrtwDskdB5mIvw+Eh9bKqUfoVNFSZvLRq+c+CEkd/n8iL20MDcjIbohP1QoP
+bNWigCAkO+ECggEBAK/ZnL4d6VYDtjoHSZRffR8AS+CqInnYwVOj+N0U4wVlLWeu
+JwoMUzuUKGYstGB09MGt3u5lOI+FvV/hwB8cM7HAY7U1nNzmgnpjX2HhA1l4nOLg
+scF5szupR0bUxS3BvjKnPnRmESEEd0GG3C+Gms+H38LOkaUmu63qms/mnFN8wvQE
+71uzcBIJGrcqkE8hsI2dipAocu4zxoOMX6toc+xLonctyoLI6U6GO+p0W/mjACDc
+gq5AxYEqCdXzo0p2R+8fIU5WOUUaH+6nzU40NLKz/lHw/zqfzDPOubzzuWoQQPrH
+s7S7u7P57STCVKF9lDFVqWyvC38W27dCpJnyQgECggEBAIo/sB4YDraPtSlEm82a
+YXE7urvZByDydbE0OH0t/r/rY+NE+3YF1fScpcQsKsotI7chiUncySMbeCTwCyfv
+HcO7/cy3eN6ugg/ZHI+NPHHzNaeSB1443QiL0zKSxKYgLAf+HRr/BrI8UgEb/CSR
+akpI5qKLTyydHvzgX6EWP3X9LpDlL83RTLEQ8qLycpHyNYK0wvJukAiN4ybp9OqS
+sGeeRGdtJmAr0tjK4dX1UN+sMYdBd+sR0K8IxPV1MLIxp8WSY0lBoMKGwDplN5hv
+5OkT7RXQLBZ1csVtzSuUjeTmncEFnEnQuIaZ4TcS7bOA9ujKBRr5LY6/xs7JDJjp
+h0ECggEAV4sDszmD7ZrFrO621d0OoMKXiJ5PNh27muej+2CUP82Tf5hxv6L1GyfY
+shERaMr4pHVmwLkWhBaJoN6I6pUgm5IIHGLxCbrKuaMb5biCg6Rw+7IwoJ6ElY1R
+iQoP89o8FfOgKKPT8wHvtUe92e58MvFs66G6dQGf3R2D5hxLGq2gEDQ8jilcIyvB
+RC9fFgFShfSczMaCvFCtUCClIXF2hi8NoYWsJlCKoz1apGeWHI8tE0ntbgjAezA1
+BCPs/HW/tGudwHlQzqGbqyg1BxjV+ogHFo+c3YB7iDmNqIMfZP1VPwk6135iKqQR
+kqoAIqDF9xFf9gDYSdBhPAEBS6zKJQ==
+-----END PRIVATE KEY-----
diff --git a/apollo-router/src/services/testdata/server.crt b/apollo-router/src/services/testdata/server.crt
new file mode 100644
index 0000000000..e88c728b78
--- /dev/null
+++ b/apollo-router/src/services/testdata/server.crt
@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFwjCCA6qgAwIBAgIUZSVRG66ZJwAcvwu2rroe8C5lVqkwDQYJKoZIhvcNAQEL
+BQAwPzELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFwaFFMMRcwFQYD
+VQQDDA5BcG9sbG8gVGVzdCBDQTAgFw0yMzA5MDgxNTI0MTJaGA8yMDUxMDEyNDE1
+MjQxMlowJjELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFwaFFMMIIC
+IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsJ8bwoJJ8OLL0U5lrhYUeWpY
+ej+eS5yibX02yWXCpzEgmailiugChwWwJrPElaarKQUYPTAlhgmFj342TWjTu8fY
+VAJO3AoHchJtkaQnyJSipagOB+nZQ936T0E6k7wdt5BfLT5ow2Enxov88YxdFdfP
+9HlYXShyXTLASCnyhgxMS8OjHO2koPQ270gV5NLVlg9+v+ZLYldDVPyLzd4TTlgW
+0PStedHFaw2i5PbLYzasF+cIb6Ld0BrknPfNaiOo3GGGs2hOGnTABTu87Y5bW3x+
+11G18p3wVwhAgHPqgbjQnZoGMq1VfLJjJcb2LTPmtGMWGBfUOIFmGWARvnhY/Txj
+ulRrWnLc+lAfj7zcoipZMkr2+JmTq6qoE9KPxdsYEuCS2aN04KK3hyppdOMIuhyF
+BHs3sWd8l5kdo1qUmunawkQrBoG2QJZgJbiOFC/ESp5Z3cGVxrIRPZ4olmP0/S5B
+8DkPyLcxjwJeinQU294o+5caJb/G53umVDG9TZb5BF+SWfQOlKnCU/NLupyEP0ZA
+FgtKmn2ztzcLTamCnqfF5LwkM0AiqXpavIF9EgRFuJ/D80PFRmPtxKxlPhBvgfD/
+1U1dWI2XO97Gp3a70RAdJhDkAKIQQesPFHFezb9ExT7BcLJCkmXy21oY4Xf+nudf
+j0ts50yyZcmmFF0Yig0CAwEAAaOBzDCByTAdBgNVHQ4EFgQUgv2yxjcZxbXqJBAS
+iv1MdiesIo4wegYDVR0jBHMwcYAUs+qeVmPP47zgP10f2MSJN2Og8xahQ6RBMD8x
+CzAJBgNVBAYTAkZSMRcwFQYDVQQKDA5BcG9sbG8gR3JhcGhRTDEXMBUGA1UEAwwO
+QXBvbGxvIFRlc3QgQ0GCFBYRF6Hac2FRkP9y5VmZ8PqIDwU3MAsGA1UdDwQEAwIC
+/DAUBgNVHREEDTALgglsb2NhbGhvc3QwCQYDVR0SBAIwADANBgkqhkiG9w0BAQsF
+AAOCAgEAEx+/UKoPbbUVA8oGPaEeyFIZdS33zD/ZL+NCNISXXKhXeNTrzlMirQVa
+2OkywIU7IWtMPwDSsM+GfO1NB3Ik8dcu+zw4QPeRMUeO6K7cHGx5AJLULq6HRKRb
++0/9fLOXxQUDoxElIN8zNqAGUsGGQGbbKGjodPiGfOnatjhCJccyGHPbq8Jek//u
+SbfpRlwbBKyKL59lLXs8zjCrbC2Teal3/KhWt1/H3dAksLYTWDPmVQooq3ezwzek
+DZh7ENGE81/2ipLPuHo4D04GGkWemLQ0h0vLmnIHQQKUscMP1Xj8hMDi2Rj/YfB0
+bf2Kle2KXJKkAOIn5n/SZrDciVydmYZEjZP7nUeB2dDwvFq+hbI6BRt94f2tAkYB
+94pLWSIG4CfGI99sRHU1WwT4F1VCUJ3pdGXl+02/mSNUCKRQnv/3OiSgo2khDL1O
+FxempDb0N9qPL2vftDhdBiPBPabvwPMGW9jRj1RHpxnTR2U8iY8cHc5E3s6N3BoT
+AElWnH/E6v6Nex74C/vw3l4rg6Ym5LUJ/Kjh6OecHnnAP2IBxFFXgl/cyMJOiOby
+Vhc5w5ySMag/S79nEjrcQzPaLPEtBQ/WUmj7mH99lBgcsJK6SyHp3twO5fETEGbw
+3m/X7mTXFhxr2MBPUH8VrIC/hJtH3ni12KmiBlYJHbDaVwm4+G8=
+-----END CERTIFICATE-----
diff --git a/apollo-router/src/services/testdata/server.csr b/apollo-router/src/services/testdata/server.csr
new file mode 100644
index 0000000000..0e2dc2723f
--- /dev/null
+++ b/apollo-router/src/services/testdata/server.csr
@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIEazCCAlMCAQAwJjELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFw
+aFFMMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsJ8bwoJJ8OLL0U5l
+rhYUeWpYej+eS5yibX02yWXCpzEgmailiugChwWwJrPElaarKQUYPTAlhgmFj342
+TWjTu8fYVAJO3AoHchJtkaQnyJSipagOB+nZQ936T0E6k7wdt5BfLT5ow2Enxov8
+8YxdFdfP9HlYXShyXTLASCnyhgxMS8OjHO2koPQ270gV5NLVlg9+v+ZLYldDVPyL
+zd4TTlgW0PStedHFaw2i5PbLYzasF+cIb6Ld0BrknPfNaiOo3GGGs2hOGnTABTu8
+7Y5bW3x+11G18p3wVwhAgHPqgbjQnZoGMq1VfLJjJcb2LTPmtGMWGBfUOIFmGWAR
+vnhY/TxjulRrWnLc+lAfj7zcoipZMkr2+JmTq6qoE9KPxdsYEuCS2aN04KK3hypp
+dOMIuhyFBHs3sWd8l5kdo1qUmunawkQrBoG2QJZgJbiOFC/ESp5Z3cGVxrIRPZ4o
+lmP0/S5B8DkPyLcxjwJeinQU294o+5caJb/G53umVDG9TZb5BF+SWfQOlKnCU/NL
+upyEP0ZAFgtKmn2ztzcLTamCnqfF5LwkM0AiqXpavIF9EgRFuJ/D80PFRmPtxKxl
+PhBvgfD/1U1dWI2XO97Gp3a70RAdJhDkAKIQQesPFHFezb9ExT7BcLJCkmXy21oY
+4Xf+nudfj0ts50yyZcmmFF0Yig0CAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQAA
+0xxj8WjL8rOM2Q/eHCTXZfAZBBTUFNsDrKCjuVUc8R7OCelOd4kjIarS/TU34+kB
+UH9oVKv485pSiDJGO38YFPLxZKTvBOCO81LjcvE4oZ9G56yu1wfPGp+quiLBfCZy
+j0UmdlxHmqrsSB2Ja2LKdCjThO57UROFfNP8f+gF70JpEPPsuOiUnGZGOCI04BEg
+FPhpbqQ/p4F1LuGGmJIxpC0+Cl2umAIPNgTnINvTknhfgoOnM0jars3xIWENXwTQ
+ojOPxbXp84+VEtKIYDzqqV+U41nsJWkIz80DucLBkgj9aQ4ysnpIvhoLFQ80jkTv
+Sa0s+fLQFndMSrWEYke9Pa3012ARJea1H9oOZsrV3Krxd9Ep9vGbuLZ+bR04uIVG
+LkMeg7XH6vguU9+QuitmIpb+r+D9XFcALK10OcAgTiavTzQntvdMKsO4xD0vkzWP
+Yju2haP9i8/vKtGXn227OVnuHy9YK3lNLPD2hz8B4Frl9jgXHrJUQ6N2NFFsL+eE
+jrCxtCGm0OGaeQRnUkrAEknr1gfGVgFs5OD6D8plHsUJTAkiPyw/xb8/LsFm1rj4
+oV0iVKFlVll1bNQL6NIa7EL4fXzro9y0wKNTIqbOz5QBEMGqCDme+Y7llhSd336W
+33NMdEdMWgUkez0EaGoEoTvvSViAB5655HpRac039Q==
+-----END CERTIFICATE REQUEST-----
diff --git a/apollo-router/src/services/testdata/server.ext b/apollo-router/src/services/testdata/server.ext
new file mode 100644
index 0000000000..c3cafab58d
--- /dev/null
+++ b/apollo-router/src/services/testdata/server.ext
@@ -0,0 +1,6 @@
+subjectKeyIdentifier   = hash
+authorityKeyIdentifier = keyid:always,issuer:always
+#basicConstraints       = CA:TRUE
+keyUsage               = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment, keyAgreement, keyCertSign
+subjectAltName         = DNS:localhost
+issuerAltName          = issuer:copy
diff --git a/apollo-router/src/services/testdata/server.key b/apollo-router/src/services/testdata/server.key
new file mode 100644
index 0000000000..c24c5bd9bb
--- /dev/null
+++ b/apollo-router/src/services/testdata/server.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCwnxvCgknw4svR
+TmWuFhR5alh6P55LnKJtfTbJZcKnMSCZqKWK6AKHBbAms8SVpqspBRg9MCWGCYWP
+fjZNaNO7x9hUAk7cCgdyEm2RpCfIlKKlqA4H6dlD3fpPQTqTvB23kF8tPmjDYSfG
+i/zxjF0V18/0eVhdKHJdMsBIKfKGDExLw6Mc7aSg9DbvSBXk0tWWD36/5ktiV0NU
+/IvN3hNOWBbQ9K150cVrDaLk9stjNqwX5whvot3QGuSc981qI6jcYYazaE4adMAF
+O7ztjltbfH7XUbXynfBXCECAc+qBuNCdmgYyrVV8smMlxvYtM+a0YxYYF9Q4gWYZ
+YBG+eFj9PGO6VGtactz6UB+PvNyiKlkySvb4mZOrqqgT0o/F2xgS4JLZo3TgoreH
+Kml04wi6HIUEezexZ3yXmR2jWpSa6drCRCsGgbZAlmAluI4UL8RKnlndwZXGshE9
+niiWY/T9LkHwOQ/ItzGPAl6KdBTb3ij7lxolv8bne6ZUMb1NlvkEX5JZ9A6UqcJT
+80u6nIQ/RkAWC0qafbO3NwtNqYKep8XkvCQzQCKpelq8gX0SBEW4n8PzQ8VGY+3E
+rGU+EG+B8P/VTV1YjZc73sandrvREB0mEOQAohBB6w8UcV7Nv0TFPsFwskKSZfLb
+Whjhd/6e51+PS2znTLJlyaYUXRiKDQIDAQABAoICACGqclQlGYr87/S5iNAiDU0H
+ZE4JCldHR/6uu16wMuZDiv8/Ei7kAndSMj1uTq5n0oBDJFelccGW1wp++ELRFGra
+sPVyay2uC7VGpfP44kxtaINDAUmo3Wz+RtQMjunSFagDsOCi4L09K6PA0zoHk0ay
+mET1TE6VgxcYsWYP+NZsSa63X8QH2DWF12LMzr+LIIM1+OcDMqNQFtmAOfXHus5Y
+JCE4QkjSH1tEgiQ6lpIWvb7gns8IsFcWiTUSVulAWtOHE3HnE9wfHOridCSj2e4a
+UOMiBBh22nT8ca8KaUzIuZsQ3NWUDJN0PUnaHtqG5hbEPFXjVORgZeG4WimALxXk
+9xZkecR1AAlStEu4uSqIPfxtxWiR1NlVzIUfrA+yfxqOid/jNdSfMekkuVfbJ5XC
+lkY6F3i1bNkwMOzZDjfEzVXXoKrqb3nyFFO5548U816hYFLlw5paVYS3v/ut7KCz
+mn3ugsAbeBp69kvuN17HgdSLG53chw7tOGs6DenRIc2fjIjOe8K5x1MLaxZATGJa
+Gw4GW41rpHA67A6EHME6WTBDHio2vJTJeOtlCbv2rEv0fp0Wl8wdhZDM+5eEahii
+Oad2utTpyGwyXgI1XvMr38v05qsJjLVqpDTsjlbvQNJzuyn73j+OShmTSzuHN4NU
+naX5Tanvu8E/P8oETi6BAoIBAQDXV6dcz11ROy+gwG8SMrXrF5LK3dkFt0xe1ha6
+ZXuQFXtkz9lceHXuR86I+BFW4V5oGDs1Q82qX5fPA66RkyfLseU9r4OLExZq5vu9
+DqeRFB9q0lavNGm3Dlf91/37f7qntqw4ytOEtyTf+1ehHbU3I14Vq2MGLFD4vj7A
+6ojm7rUSIbNKLajlmIWymT5T+DOvnIXsZjkAFLG9TF2yQswtqcv+qcaAGIrmAGgG
+eFIReBXb7wQ9OBSWKSSu91iqQ687jV4i18NR52L9Wot/0+lCxxkKrsmcSRcqFKFn
++GO7J/D+QDNyWyCbaz/uxuc2gbGVa+q32n1V05iUeuydcGUVAoIBAQDR9+wKDMOH
+rXdYSJ4QSpqn17NstwlL8SDcYbVdU18PxMNOGB7pqoeevmhoVtFhYCV5+lYDMTrG
+bR1cNwj95aTlKofWETkf5we1od4Hue6zIYuV6vRhWbzEQaAWbDtUiE4+LckakYxk
+ETQ3MUzMgQIdn6P2UnXU9rx1efxOdBe+zZf7/0acq15AEEudSEwDotx8NwdclcYW
+YTd966o0K3ctlHXvwnOJD4oV1ZlMG2+Gbip+wiErAtn791F9dlQ5yoAS7kqQscX+
+ghDkffep0/mdPUx05hzeb7k8O0xpfKOcLTBLs7L7Xs7mnMI+3+Videg8NX7j/o0a
+FXUGlkRv5L8ZAoIBAHqAcMMJbAqj4spN7mGp23drp6QkZCsuRRHGNJPA23Px5IpK
++1nxZRUyYDnm9OY5utz4u+XnNn5vdwrbC1nhHshv4Psgd3KMXzpdQGQijAyqJ906
+3vk4ldL/I+tSDjgeR27VKk/8giuDjESm1Qjgv5AoXINzRhfi6gU3JLfWNB87OGKi
+LT5Lj0NdauFY7+4QykZigoUi+dYj76DCaqLYARMQ/u1WY4fHbmy9WnAjznwFllnR
+NNYmqv9pVGb8asIhUvRLzXuAxqo3OUbrVWRYht/oJ0S+fg6cPcNAjTl6QjHb3Oqm
+Vx383SXM80fAexO03/Miuxnv/1Dt2GTPMP+3SqkCggEBAMJ36DNKrRKN1Dp9T4Bx
+SJppynHHeSHQHAphqNPeMu5s5CMBbignXGLnodX5iwWwQiWeeWmNgA1l5EqTYo2d
+00K5SIQ5AXqNxPMyD22ChRV93UWmDl/X+GChE3+REMZzVFx8lYU5BxlrYbw4uhOy
+7N25DOhIaYAsYlJAADI9jfyXeMvaHHFguvOE5DOxUU4Q2jtl0DitM4Fo2zeSPNsC
+4ufhV/YPWGi6br9YTRpTUtDkWhF7BbNLec/Hub8FCG9aIZLnB0LKlEHr/29RSEAP
+52H980FWG6jdO2gQ1+3flb1BkTicIAp3PhQTXklTn6W45LZXKcZK7i8ZboECTjKA
+X4kCggEBAKZwBpNwroJ8h3fNIufHGu5cGDaQQ4piGQZ/RQ7bowDgLGm1ONG6TYso
+nroJrzSu1SxYiGJh2T2E1uYudVT1v6ebDrt2IgZdrajOrTbFi14Z50nkaYJLMfTc
+fg3abp7v4E79TZP67npxBOeLzNOoCS5HrSwiaNiMbGkUiOEBUwV1seRMcZX1yc1n
+xGFA+C4WOp4p8sJ865Nkipj2hE0x2PRtvd6oa/ZIMVNXXZFXTVASUc9ZwCgbGdM8
++iaHqsPzfVUL+o6YmzDV+7fuFI8MFoXRw9XRqPYfRPQnLl8fO/YW166qvvVh1Zhx
+4dWw/oAnLAPKVv38o9bOcPHDGSLLAhU=
+-----END PRIVATE KEY-----
diff --git a/apollo-router/src/services/testdata/server_self_signed.crt b/apollo-router/src/services/testdata/server_self_signed.crt
new file mode 100644
index 0000000000..cd5d3b9f3c
--- /dev/null
+++ b/apollo-router/src/services/testdata/server_self_signed.crt
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFmTCCA4GgAwIBAgIUDJn4No8XZo+1xUEEMqpAqRi1xmUwDQYJKoZIhvcNAQEL
+BQAwJjELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFwaFFMMB4XDTIz
+MDkwODE1MjM1M1oXDTIzMTAwODE1MjM1M1owJjELMAkGA1UEBhMCRlIxFzAVBgNV
+BAoMDkFwb2xsbyBHcmFwaFFMMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
+AgEAsJ8bwoJJ8OLL0U5lrhYUeWpYej+eS5yibX02yWXCpzEgmailiugChwWwJrPE
+laarKQUYPTAlhgmFj342TWjTu8fYVAJO3AoHchJtkaQnyJSipagOB+nZQ936T0E6
+k7wdt5BfLT5ow2Enxov88YxdFdfP9HlYXShyXTLASCnyhgxMS8OjHO2koPQ270gV
+5NLVlg9+v+ZLYldDVPyLzd4TTlgW0PStedHFaw2i5PbLYzasF+cIb6Ld0BrknPfN
+aiOo3GGGs2hOGnTABTu87Y5bW3x+11G18p3wVwhAgHPqgbjQnZoGMq1VfLJjJcb2
+LTPmtGMWGBfUOIFmGWARvnhY/TxjulRrWnLc+lAfj7zcoipZMkr2+JmTq6qoE9KP
+xdsYEuCS2aN04KK3hyppdOMIuhyFBHs3sWd8l5kdo1qUmunawkQrBoG2QJZgJbiO
+FC/ESp5Z3cGVxrIRPZ4olmP0/S5B8DkPyLcxjwJeinQU294o+5caJb/G53umVDG9
+TZb5BF+SWfQOlKnCU/NLupyEP0ZAFgtKmn2ztzcLTamCnqfF5LwkM0AiqXpavIF9
+EgRFuJ/D80PFRmPtxKxlPhBvgfD/1U1dWI2XO97Gp3a70RAdJhDkAKIQQesPFHFe
+zb9ExT7BcLJCkmXy21oY4Xf+nudfj0ts50yyZcmmFF0Yig0CAwEAAaOBvjCBuzAd
+BgNVHQ4EFgQUgv2yxjcZxbXqJBASiv1MdiesIo4wYQYDVR0jBFowWIAUgv2yxjcZ
+xbXqJBASiv1MdiesIo6hKqQoMCYxCzAJBgNVBAYTAkZSMRcwFQYDVQQKDA5BcG9s
+bG8gR3JhcGhRTIIUDJn4No8XZo+1xUEEMqpAqRi1xmUwCwYDVR0PBAQDAgL8MBQG
+A1UdEQQNMAuCCWxvY2FsaG9zdDAUBgNVHRIEDTALgglsb2NhbGhvc3QwDQYJKoZI
+hvcNAQELBQADggIBACjN3nhmJD2k6i3vSDAmfLtdzuSIbRSHPUXMyv4kRS1XaRw2
+QyRhnZN+7LDoGyzI8IkSNs14xeUtixw5Nl0kFiq9cEuTvptAUNbPi4n/UuI3tSDf
+8VX/0WCoJonD5QlnQ9RPBGqbVmyzn1nDIdLyKsbUpK4igdDkj+DeLva8bO41/MMk
+e4PONDwHO4dyA00OlzyYazxK72tSDAEUomgDNvYtaNAi7uxXWnAYPkuXX77JgGxS
+hTBdMXPIUhcUCLotL5sIKj4UMk1BWUom9egL21W+M6A97M9+uCr72nktM29E8IGc
+euuqgq5bgALRthgaecsUPu/MlnUumLK9v7+osFtIAl6fGYQMvcdX90jII0QWoAdk
+uleVU8Ae18588e527clV5B+TnGjMucbxoPr4AbWR9slvdNVBB5UMHg67UsqF/Kyq
+F1WK5Zr5RD5r3yEwEmVkzQops/TmNRfmm+rFqrOMwRxpb8eW0jR4qfGPcFKfoZu3
+8xif2zKl9w21xQZ6VFVpgir8QJf0uswsHdlMRABAfiBBReP4Pg+jFFbmijajdU/i
+w46qc4whI5dbF+nAGdg3NsC8P+zeBTXQJEoF7E+M+eu35pOxi8xi46NKHsqfNNtG
+hLT0oWvdWPRH009SPHcA1fTZ347pkOeGtz94IJjTPm+HI1sHWv7avHX3U4Pr
+-----END CERTIFICATE-----
diff --git a/apollo-router/src/services/testdata/server_self_signed.csr b/apollo-router/src/services/testdata/server_self_signed.csr
new file mode 100644
index 0000000000..0e2dc2723f
--- /dev/null
+++ b/apollo-router/src/services/testdata/server_self_signed.csr
@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIEazCCAlMCAQAwJjELMAkGA1UEBhMCRlIxFzAVBgNVBAoMDkFwb2xsbyBHcmFw
+aFFMMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsJ8bwoJJ8OLL0U5l
+rhYUeWpYej+eS5yibX02yWXCpzEgmailiugChwWwJrPElaarKQUYPTAlhgmFj342
+TWjTu8fYVAJO3AoHchJtkaQnyJSipagOB+nZQ936T0E6k7wdt5BfLT5ow2Enxov8
+8YxdFdfP9HlYXShyXTLASCnyhgxMS8OjHO2koPQ270gV5NLVlg9+v+ZLYldDVPyL
+zd4TTlgW0PStedHFaw2i5PbLYzasF+cIb6Ld0BrknPfNaiOo3GGGs2hOGnTABTu8
+7Y5bW3x+11G18p3wVwhAgHPqgbjQnZoGMq1VfLJjJcb2LTPmtGMWGBfUOIFmGWAR
+vnhY/TxjulRrWnLc+lAfj7zcoipZMkr2+JmTq6qoE9KPxdsYEuCS2aN04KK3hypp
+dOMIuhyFBHs3sWd8l5kdo1qUmunawkQrBoG2QJZgJbiOFC/ESp5Z3cGVxrIRPZ4o
+lmP0/S5B8DkPyLcxjwJeinQU294o+5caJb/G53umVDG9TZb5BF+SWfQOlKnCU/NL
+upyEP0ZAFgtKmn2ztzcLTamCnqfF5LwkM0AiqXpavIF9EgRFuJ/D80PFRmPtxKxl
+PhBvgfD/1U1dWI2XO97Gp3a70RAdJhDkAKIQQesPFHFezb9ExT7BcLJCkmXy21oY
+4Xf+nudfj0ts50yyZcmmFF0Yig0CAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQAA
+0xxj8WjL8rOM2Q/eHCTXZfAZBBTUFNsDrKCjuVUc8R7OCelOd4kjIarS/TU34+kB
+UH9oVKv485pSiDJGO38YFPLxZKTvBOCO81LjcvE4oZ9G56yu1wfPGp+quiLBfCZy
+j0UmdlxHmqrsSB2Ja2LKdCjThO57UROFfNP8f+gF70JpEPPsuOiUnGZGOCI04BEg
+FPhpbqQ/p4F1LuGGmJIxpC0+Cl2umAIPNgTnINvTknhfgoOnM0jars3xIWENXwTQ
+ojOPxbXp84+VEtKIYDzqqV+U41nsJWkIz80DucLBkgj9aQ4ysnpIvhoLFQ80jkTv
+Sa0s+fLQFndMSrWEYke9Pa3012ARJea1H9oOZsrV3Krxd9Ep9vGbuLZ+bR04uIVG
+LkMeg7XH6vguU9+QuitmIpb+r+D9XFcALK10OcAgTiavTzQntvdMKsO4xD0vkzWP
+Yju2haP9i8/vKtGXn227OVnuHy9YK3lNLPD2hz8B4Frl9jgXHrJUQ6N2NFFsL+eE
+jrCxtCGm0OGaeQRnUkrAEknr1gfGVgFs5OD6D8plHsUJTAkiPyw/xb8/LsFm1rj4
+oV0iVKFlVll1bNQL6NIa7EL4fXzro9y0wKNTIqbOz5QBEMGqCDme+Y7llhSd336W
+33NMdEdMWgUkez0EaGoEoTvvSViAB5655HpRac039Q==
+-----END CERTIFICATE REQUEST-----
diff --git a/apollo-router/src/services/testdata/tls.md b/apollo-router/src/services/testdata/tls.md
new file mode 100644
index 0000000000..c5d46d74bc
--- /dev/null
+++ b/apollo-router/src/services/testdata/tls.md
@@ -0,0 +1,82 @@
+# Certificate generation
+
+## Server self signed certificate
+
+```
+openssl genrsa -out server.key 4096
+openssl req -new -key server.key -out server_self_signed.csr
+openssl x509 -req -in server_self_signed.csr -signkey server.key -out server_self_signed.crt -extfile server.ext
+```
+
+## Root certificate authority
+
+```
+openssl genrsa -out server.key 4096
+openssl req -new -x509 -days 10000 -key ca.key -out ca.crt
+
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) [AU]:FR
+State or Province Name (full name) [Some-State]:.
+Locality Name (eg, city) []:
+Organization Name (eg, company) [Internet Widgits Pty Ltd]:Apollo GraphQL
+Organizational Unit Name (eg, section) []:
+Common Name (e.g. server FQDN or YOUR name) []:Apollo Test CA
+Email Address []:
+```
+
+## Server certificate
+
+```
+openssl genrsa -out server.key 4096
+openssl req -new -key server.key -out server.csr
+openssl x509 -req -in server.csr -CA ./CA/ca.crt -CAkey ./CA/ca.key -out server.crt -CAcreateserial -days 10000 -sha256 -extfile server.ext
+Certificate request self-signature ok
+subject=C = FR, O = Apollo GraphQL, CN = router
+```
+
+## Client certificate
+
+Generate the key:
+
+```
+openssl genrsa -out client.key.pem 4096
+```
+
+Certificate signing request:
+
+```
+openssl req -new -key client.key -out client.csr
+
+You are about to be asked to enter information that will be incorporated   
+into your certificate request.                                             
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank                  
+For some fields there will be a default value,                             
+If you enter '.', the field will be left blank.                            
+-----                                                                      
+Country Name (2 letter code) [AU]:FR                                       
+State or Province Name (full name) [Some-State]:.                          
+Locality Name (eg, city) []:                                               
+Organization Name (eg, company) [Internet Widgits Pty Ltd]:Apollo GraphQL  
+Organizational Unit Name (eg, section) []:                                 
+Common Name (e.g. server FQDN or YOUR name) []:router                      
+Email Address []:                                                          
+                                                                           
+Please enter the following 'extra' attributes                              
+to be sent with your certificate request                                   
+A challenge password []:                                                   
+An optional company name []:                                               
+```
+
+Generate the certificate:
+```
+openssl x509 -req -in client.csr -CA ./CA/ca.crt -CAkey ./CA/ca.key -out client.crt -CAcreateserial -days 10000 -sha256 -extfile client.ext
+Certificate request self-signature ok
+subject=C = FR, O = Apollo GraphQL, CN = router
+```
diff --git a/docs/source/configuration/overview.mdx b/docs/source/configuration/overview.mdx
index b68ecb664d..b96f4fcc2a 100644
--- a/docs/source/configuration/overview.mdx
+++ b/docs/source/configuration/overview.mdx
@@ -567,7 +567,7 @@ The router expects the file referenced in the `certificate_chain` value to be a
 
 The router verifies TLS connections to subgraphs using the list of certificate authorities the system provides. You can override this list with a combination of global and per-subgraph settings:
 
-```yaml title="router.yaml"
+```yaml
 tls:
   subgraph:
     # Use these certificate authorities unless overridden per-subgraph
@@ -605,6 +605,26 @@ issuerAltName          = issuer:copy
 > Make sure to change the `subjectAltName` field to the subgraph's name.
 This produces the file as `server.crt` which can be used in `certificate_authorities`.
 
+#### TLS client authentication for subgraph requests
+
+The router support mutual TLS authentication (mTLS) with the subgraphs. This means that it can authenticate itself to the subgraph using a certificate chain and a cryptographic key. It can be configured as follows:
+
+```yaml
+tls:
+  subgraph:
+    # Use these certificates and key unless overridden per-subgraph
+    all:
+      client_authentication:
+        certificate_chain: ${file./path/to/certificate_chain.pem}
+        key: ${file./path/to/key.pem}
+    # Override global setting for individual subgraphs
+    subgraphs:
+      products:
+        client_authentication:
+          certificate_chain: ${file./path/to/certificate_chain.pem}
+          key: ${file./path/to/key.pem}
+```
+
 ### Request limits
 
 > **Request limits are currently in [preview](/resources/product-launch-stages#preview).**

From 3af25efba312e0287a023e8562823903b0aee82a Mon Sep 17 00:00:00 2001
From: Geoffroy Couprie 
Date: Wed, 20 Sep 2023 17:53:16 +0200
Subject: [PATCH 30/51] reintroduce rhai json functions (#3785)

Follow up to https://github.com/apollographql/router/pull/3782
---
 .../feat_geal_reintroduce_rhai_json.md        |  5 ++++
 apollo-router/src/plugins/rhai/engine.rs      | 30 +++++++++++++++++++
 docs/source/customizations/rhai-api.mdx       |  6 ++--
 3 files changed, 38 insertions(+), 3 deletions(-)
 create mode 100644 .changesets/feat_geal_reintroduce_rhai_json.md

diff --git a/.changesets/feat_geal_reintroduce_rhai_json.md b/.changesets/feat_geal_reintroduce_rhai_json.md
new file mode 100644
index 0000000000..f76e238eca
--- /dev/null
+++ b/.changesets/feat_geal_reintroduce_rhai_json.md
@@ -0,0 +1,5 @@
+### JSON encoding and decoding in Rhai ([PR #3785](https://github.com/apollographql/router/pull/3785))
+
+It is now possible to encode or decode JSON from Rhai scripts using `json::encode` and `json::decode`
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3785
\ No newline at end of file
diff --git a/apollo-router/src/plugins/rhai/engine.rs b/apollo-router/src/plugins/rhai/engine.rs
index b4f55d1653..8713ed4d91 100644
--- a/apollo-router/src/plugins/rhai/engine.rs
+++ b/apollo-router/src/plugins/rhai/engine.rs
@@ -102,6 +102,32 @@ mod router_base64 {
     }
 }
 
+#[export_module]
+mod router_json {
+    pub(crate) type Object = crate::json_ext::Object;
+    pub(crate) type Value = crate::json_ext::Value;
+
+    #[rhai_fn(name = "to_string", pure)]
+    pub(crate) fn object_to_string(x: &mut Object) -> String {
+        format!("{x:?}")
+    }
+
+    #[rhai_fn(name = "to_string", pure)]
+    pub(crate) fn value_to_string(x: &mut Value) -> String {
+        format!("{x:?}")
+    }
+
+    #[rhai_fn(pure, return_raw)]
+    pub(crate) fn encode(input: &mut Dynamic) -> Result> {
+        serde_json::to_string(input).map_err(|e| e.to_string().into())
+    }
+
+    #[rhai_fn(pure, return_raw)]
+    pub(crate) fn decode(input: &mut ImmutableString) -> Result> {
+        serde_json::from_str(input).map_err(|e| e.to_string().into())
+    }
+}
+
 #[export_module]
 mod router_expansion {
     pub(crate) type Expansion = expansion::Expansion;
@@ -1464,6 +1490,8 @@ impl Rhai {
         combine_with_exported_module!(&mut module, "context", router_context);
 
         let base64_module = exported_module!(router_base64);
+        let json_module = exported_module!(router_json);
+
         let expansion_module = exported_module!(router_expansion);
 
         // Share main so we can move copies into each closure as required for logging
@@ -1487,6 +1515,8 @@ impl Rhai {
             .register_global_module(module.into())
             // Register our base64 module (not global)
             .register_static_module("base64", base64_module.into())
+            // Register our json module (not global)
+            .register_static_module("json", json_module.into())
             // Register our expansion module (not global)
             // Hide the fact that it is an expansion module by calling it "env"
             .register_static_module("env", expansion_module.into())
diff --git a/docs/source/customizations/rhai-api.mdx b/docs/source/customizations/rhai-api.mdx
index add590aefb..11a2c25ac7 100644
--- a/docs/source/customizations/rhai-api.mdx
+++ b/docs/source/customizations/rhai-api.mdx
@@ -195,16 +195,16 @@ fn supergraph_service(service) {
 
 ## json encode/decode strings
 
-Your Rhai customization can use the functions `json_encode()` and `json_decode()` to convert Rhai objects to/from valid JSON encoded strings. Both functions can fail, so always handle exceptions when using them.
+Your Rhai customization can use the functions `json::encode()` and `json::decode()` to convert Rhai objects to/from valid JSON encoded strings. Both functions can fail, so always handle exceptions when using them.
 
 ```rhai
 fn router_service(service) {
     let original = `{"valid":"object"}`;
     try {
-        let encoded = json_decode(original);
+        let encoded = json::decode(original);
         // encoded is a Rhai object, with a property (or key) named valid with a String value of "object"
         print(`encoded.valid: ${encoded.valid}`);
-        let and_back = json_encode(encoded);
+        let and_back = json::encode(encoded);
         // and_back will be a string == original.
         if and_back != original {
             throw "something has gone wrong";

From a1462cd0991c8a38fe6c6982563c6125338445ac Mon Sep 17 00:00:00 2001
From: Gary Pennington 
Date: Thu, 21 Sep 2023 10:30:39 +0100
Subject: [PATCH 31/51] Disable helm check for missing kubernetes versions
 until supported by kubeconform (#3867)

re-enables the CI build pipeline. We'll restore these version checks in
the future by reverting this.
---
 .circleci/config.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 63decfa09b..bd7fa34a99 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -399,6 +399,10 @@ commands:
             # Use helm to template our chart against all kube versions
             TEMPLATE_DIR=$(mktemp -d)
             for kube_version in ${CURRENT_KUBE_VERSIONS}; do
+              # Skip 1.25.14, 1.26.9, 1.27.6 and 1.28.2 until supported by kubeconform
+              if [[ "${kube_version}" == "1.25.14" ]] || [[ "${kube_version}" == "1.26.9" ]] || [[ "${kube_version}" == "1.27.6" ]] || [[ "${kube_version}" == "1.28.2" ]]; then
+                continue
+              fi
               # Use helm to template our chart against kube_version
               helm template --kube-version "${kube_version}" router helm/chart/router --set autoscaling.enabled=true > "${TEMPLATE_DIR}/router-${kube_version}.yaml"
 

From 99d2c3546c98a72423828b80e6fb76d835ed0bf2 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 21 Sep 2023 09:32:53 +0000
Subject: [PATCH 32/51] chore(deps): update rust crate insta to 1.32.0

---
 Cargo.lock               | 4 ++--
 apollo-router/Cargo.toml | 2 +-
 xtask/Cargo.lock         | 4 ++--
 xtask/Cargo.toml         | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e5bcce7488..c0a5684775 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3287,9 +3287,9 @@ dependencies = [
 
 [[package]]
 name = "insta"
-version = "1.31.0"
+version = "1.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0770b0a3d4c70567f0d58331f3088b0e4c4f56c9b8d764efe654b4a5d46de3a"
+checksum = "a3e02c584f4595792d09509a94cdb92a3cef7592b1eb2d9877ee6f527062d0ea"
 dependencies = [
  "console 0.15.7",
  "lazy_static",
diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml
index 3a7d621301..b1295471e6 100644
--- a/apollo-router/Cargo.toml
+++ b/apollo-router/Cargo.toml
@@ -262,7 +262,7 @@ axum = { version = "0.6.20", features = [
 ecdsa = { version = "0.15.1", features = ["signing", "pem", "pkcs8"] }
 fred = { version = "6.3.2", features = ["enable-rustls", "no-client-setname"] }
 futures-test = "0.3.28"
-insta = { version = "1.31.0", features = ["json", "redactions", "yaml"] }
+insta = { version = "1.32.0", features = ["json", "redactions", "yaml"] }
 maplit = "1.0.2"
 memchr = { version = "2.6.3", default-features = false }
 mockall = "0.11.4"
diff --git a/xtask/Cargo.lock b/xtask/Cargo.lock
index f2b6342c9e..87884db85e 100644
--- a/xtask/Cargo.lock
+++ b/xtask/Cargo.lock
@@ -758,9 +758,9 @@ dependencies = [
 
 [[package]]
 name = "insta"
-version = "1.31.0"
+version = "1.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0770b0a3d4c70567f0d58331f3088b0e4c4f56c9b8d764efe654b4a5d46de3a"
+checksum = "a3e02c584f4595792d09509a94cdb92a3cef7592b1eb2d9877ee6f527062d0ea"
 dependencies = [
  "console",
  "lazy_static",
diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml
index ad36d2b3b5..a5630d583c 100644
--- a/xtask/Cargo.toml
+++ b/xtask/Cargo.toml
@@ -46,4 +46,4 @@ base64 = "0.21"
 zip = { version = "0.6", default-features = false }
 
 [dev-dependencies]
-insta = { version = "1.31.0", features = ["json", "redactions", "yaml"] }
+insta = { version = "1.32.0", features = ["json", "redactions", "yaml"] }

From 9efaacfdb3efa08755b819a1799ea8c618adf68a Mon Sep 17 00:00:00 2001
From: Gary Pennington 
Date: Thu, 21 Sep 2023 13:43:56 +0100
Subject: [PATCH 33/51] Introduce OneShotAsyncCheckpoint (#3819)

The existing AsynCheckpoint requires `Clone` and thus introduces a need
for Service Buffering that reduces the performance and resiliance of the
router.

This new service set of Services, Layers and utility functions removes
the requirement for `Clone` and thus the requirement for service
buffering.

Existing uses of AsyncCheckpoint within the router are replaced with
OneShotAsyncCheckpoint along with the requirement to `buffer()` such
services.


---

**Checklist**

Complete the checklist (and note appropriate exceptions) before the PR
is marked ready-for-review.

- [x] Changes are compatible[^1]
- [x] Documentation[^2] completed
- [x] Performance impact assessed and acceptable
- Tests added and passing[^3]
    - [x] Unit Tests
    - [ ] Integration Tests
    - [ ] Manual Tests

**Exceptions**

*Note any exceptions here*

**Notes**

[^1]: It may be appropriate to bring upcoming changes to the attention
of other (impacted) groups. Please endeavour to do this before seeking
PR approval. The mechanism for doing this will vary considerably, so use
your judgement as to how and when to do this.
[^2]: Configuration is an important part of many changes. Where
applicable please try to document configuration examples.
[^3]: Tick whichever testing boxes are applicable. If you are adding
Manual Tests, please document the manual testing (extensively) in the
Exceptions.
---
 .../feat_garypen_one_shot_async_check.md      |  11 +
 .../plugin/src/plugins/{{snake_name}}.rs      |   3 +-
 apollo-router/src/layers/async_checkpoint.rs  | 278 ++++++++++++++++++
 apollo-router/src/layers/mod.rs               |  58 ++++
 apollo-router/src/plugins/coprocessor/mod.rs  |   8 +-
 .../src/plugins/coprocessor/supergraph.rs     |   5 +-
 .../layers/allow_only_http_post_mutations.rs  |  57 ++--
 docs/source/customizations/native.mdx         |   1 +
 .../rust/src/allow_client_id_from_file.rs     |  16 +-
 9 files changed, 377 insertions(+), 60 deletions(-)
 create mode 100644 .changesets/feat_garypen_one_shot_async_check.md

diff --git a/.changesets/feat_garypen_one_shot_async_check.md b/.changesets/feat_garypen_one_shot_async_check.md
new file mode 100644
index 0000000000..61298d848c
--- /dev/null
+++ b/.changesets/feat_garypen_one_shot_async_check.md
@@ -0,0 +1,11 @@
+### Introduce OneShotAsyncCheckpoint ([PR #3819](https://github.com/apollographql/router/pull/3819))
+
+The existing AsynCheckpoint requires `Clone` and thus introduces a need for Service Buffering which reduces the performance and resiliance of the router.
+
+This new set of Services, Layers and utility functions removes the requirement for `Clone` and thus the requirement for service buffering.
+
+Existing uses of AsyncCheckpoint within the router are replaced with OneShotAsyncCheckpoint along with the requirement to `buffer()` such services.
+
+If you have a custom plugin that makes use of `AsyncCheckpoint`, we encourage you to migrate to `OneShotAsyncCheckpoint` and thus reduce the requirement for service buffering from your router.
+
+By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3819
\ No newline at end of file
diff --git a/apollo-router-scaffold/templates/plugin/src/plugins/{{snake_name}}.rs b/apollo-router-scaffold/templates/plugin/src/plugins/{{snake_name}}.rs
index fa994f23c5..49a11b8d91 100644
--- a/apollo-router-scaffold/templates/plugin/src/plugins/{{snake_name}}.rs
+++ b/apollo-router-scaffold/templates/plugin/src/plugins/{{snake_name}}.rs
@@ -113,11 +113,10 @@ impl Plugin for {{pascal_name}} {
     ) -> supergraph::BoxService {
 
         ServiceBuilder::new()
-                    .checkpoint_async(|request : supergraph::Request| async {
+                    .oneshot_checkpoint_async(|request : supergraph::Request| async {
                         // Do some async call here to auth, and decide if to continue or not.
                         Ok(ControlFlow::Continue(request))
                     })
-                    .buffered()
                     .service(service)
                     .boxed()
     }
diff --git a/apollo-router/src/layers/async_checkpoint.rs b/apollo-router/src/layers/async_checkpoint.rs
index 79226ff976..a05bb6a78a 100644
--- a/apollo-router/src/layers/async_checkpoint.rs
+++ b/apollo-router/src/layers/async_checkpoint.rs
@@ -13,6 +13,7 @@ use std::marker::PhantomData;
 use std::ops::ControlFlow;
 use std::pin::Pin;
 use std::sync::Arc;
+use std::task::Poll;
 
 use futures::future::BoxFuture;
 use futures::Future;
@@ -67,6 +68,83 @@ where
     }
 }
 
+/// [`Service`] for OneShot (single use) Asynchronous Checkpoints. See [`ServiceBuilderExt::oneshot_checkpoint_async()`](crate::layers::ServiceBuilderExt::oneshot_checkpoint_async()).
+#[allow(clippy::type_complexity)]
+pub struct OneShotAsyncCheckpointService
+where
+    Request: Send + 'static,
+    S: Service + Send + 'static,
+    >::Response: Send + 'static,
+    >::Future: Send + 'static,
+    Fut: Future>::Response, Request>, BoxError>>,
+{
+    inner: Option,
+    checkpoint_fn: Arc Fut + Send + Sync + 'static>>>,
+}
+
+impl OneShotAsyncCheckpointService
+where
+    Request: Send + 'static,
+    S: Service + Send + 'static,
+    >::Response: Send + 'static,
+    >::Future: Send + 'static,
+    Fut: Future>::Response, Request>, BoxError>>,
+{
+    /// Create an `OneShotAsyncCheckpointLayer` from a function that takes a Service Request and returns a `ControlFlow`
+    pub fn new(checkpoint_fn: F, service: S) -> Self
+    where
+        F: Fn(Request) -> Fut + Send + Sync + 'static,
+    {
+        Self {
+            checkpoint_fn: Arc::new(Box::pin(checkpoint_fn)),
+            inner: Some(service),
+        }
+    }
+}
+
+impl Service for OneShotAsyncCheckpointService
+where
+    Request: Send + 'static,
+    S: Service + Send + 'static,
+    >::Response: Send + 'static,
+    >::Future: Send + 'static,
+    Fut: Future>::Response, Request>, BoxError>>
+        + Send
+        + 'static,
+{
+    type Response = >::Response;
+
+    type Error = BoxError;
+
+    type Future = BoxFuture<'static, Result>;
+
+    fn poll_ready(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll> {
+        // Return an error if we no longer have an inner service
+        match self.inner.as_mut() {
+            Some(inner) => inner.poll_ready(cx),
+            None => Poll::Ready(Err("One shot must only be called once".into())),
+        }
+    }
+
+    fn call(&mut self, req: Request) -> Self::Future {
+        let checkpoint_fn = Arc::clone(&self.checkpoint_fn);
+        let inner = self
+            .inner
+            .take()
+            .expect("One shot must only be called once");
+        Box::pin(async move {
+            match (checkpoint_fn)(req).await {
+                Ok(ControlFlow::Break(response)) => Ok(response),
+                Ok(ControlFlow::Continue(request)) => inner.oneshot(request).await,
+                Err(error) => Err(error),
+            }
+        })
+    }
+}
+
 /// [`Service`] for Asynchronous Checkpoints. See [`ServiceBuilderExt::checkpoint_async()`](crate::layers::ServiceBuilderExt::checkpoint_async()).
 #[allow(clippy::type_complexity)]
 pub struct AsyncCheckpointService
@@ -137,6 +215,52 @@ where
     }
 }
 
+/// [`Layer`] for OneShot (single use) Asynchronous Checkpoints. See [`ServiceBuilderExt::oneshot_checkpoint_async()`](crate::layers::ServiceBuilderExt::oneshot_checkpoint_async()).
+#[allow(clippy::type_complexity)]
+pub struct OneShotAsyncCheckpointLayer
+where
+    S: Service + Send + 'static,
+    Fut: Future>::Response, Request>, BoxError>>,
+{
+    checkpoint_fn: Arc Fut + Send + Sync + 'static>>>,
+    phantom: PhantomData, // XXX: The compiler can't detect that S is used in the Future...
+}
+
+impl OneShotAsyncCheckpointLayer
+where
+    S: Service + Send + 'static,
+    Fut: Future>::Response, Request>, BoxError>>,
+{
+    /// Create an `OneShotAsyncCheckpointLayer` from a function that takes a Service Request and returns a `ControlFlow`
+    pub fn new(checkpoint_fn: F) -> Self
+    where
+        F: Fn(Request) -> Fut + Send + Sync + 'static,
+    {
+        Self {
+            checkpoint_fn: Arc::new(Box::pin(checkpoint_fn)),
+            phantom: PhantomData,
+        }
+    }
+}
+
+impl Layer for OneShotAsyncCheckpointLayer
+where
+    S: Service + Send + 'static,
+    >::Future: Send,
+    Request: Send + 'static,
+    >::Response: Send + 'static,
+    Fut: Future>::Response, Request>, BoxError>>,
+{
+    type Service = OneShotAsyncCheckpointService;
+
+    fn layer(&self, service: S) -> Self::Service {
+        OneShotAsyncCheckpointService {
+            checkpoint_fn: Arc::clone(&self.checkpoint_fn),
+            inner: Some(service),
+        }
+    }
+}
+
 #[cfg(test)]
 mod async_checkpoint_tests {
     use tower::BoxError;
@@ -286,4 +410,158 @@ mod async_checkpoint_tests {
 
         assert_eq!(actual_error, expected_error)
     }
+
+    #[tokio::test]
+    async fn test_service_builder_oneshot() {
+        let expected_label = "from_mock_service";
+
+        let mut execution_service = MockExecutionService::new();
+        execution_service
+            .expect_call()
+            .times(1)
+            .returning(move |req: ExecutionRequest| {
+                Ok(ExecutionResponse::fake_builder()
+                    .label(expected_label.to_string())
+                    .context(req.context)
+                    .build()
+                    .unwrap())
+            });
+
+        let service_stack = ServiceBuilder::new()
+            .oneshot_checkpoint_async(|req: ExecutionRequest| async {
+                Ok(ControlFlow::Continue(req))
+            })
+            .service(execution_service);
+
+        let request = ExecutionRequest::fake_builder().build();
+
+        let actual_label = service_stack
+            .oneshot(request)
+            .await
+            .unwrap()
+            .next_response()
+            .await
+            .unwrap()
+            .label
+            .unwrap();
+
+        assert_eq!(actual_label, expected_label)
+    }
+
+    #[tokio::test]
+    #[should_panic]
+    async fn test_service_builder_buffered_oneshot() {
+        let expected_label = "from_mock_service";
+
+        let mut execution_service = MockExecutionService::new();
+        execution_service
+            .expect_call()
+            .times(1)
+            .returning(move |req: ExecutionRequest| {
+                Ok(ExecutionResponse::fake_builder()
+                    .label(expected_label.to_string())
+                    .context(req.context)
+                    .build()
+                    .unwrap())
+            });
+
+        let mut service_stack = ServiceBuilder::new()
+            .oneshot_checkpoint_async(|req: ExecutionRequest| async {
+                Ok(ControlFlow::Continue(req))
+            })
+            .buffered()
+            .service(execution_service);
+
+        let request = ExecutionRequest::fake_builder().build();
+        let request_again = ExecutionRequest::fake_builder().build();
+
+        let _ = service_stack.call(request).await.unwrap();
+        // Trying to use the service again should cause a panic
+        let _ = service_stack.call(request_again).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_continue_oneshot() {
+        let expected_label = "from_mock_service";
+        let mut router_service = MockExecutionService::new();
+        router_service
+            .expect_call()
+            .times(1)
+            .returning(move |_req| {
+                Ok(ExecutionResponse::fake_builder()
+                    .label(expected_label.to_string())
+                    .build()
+                    .unwrap())
+            });
+
+        let service_stack =
+            OneShotAsyncCheckpointLayer::new(|req| async { Ok(ControlFlow::Continue(req)) })
+                .layer(router_service);
+
+        let request = ExecutionRequest::fake_builder().build();
+
+        let actual_label = service_stack
+            .oneshot(request)
+            .await
+            .unwrap()
+            .next_response()
+            .await
+            .unwrap()
+            .label
+            .unwrap();
+
+        assert_eq!(actual_label, expected_label)
+    }
+
+    #[tokio::test]
+    async fn test_return_oneshot() {
+        let expected_label = "returned_before_mock_service";
+        let router_service = MockExecutionService::new();
+
+        let service_stack = OneShotAsyncCheckpointLayer::new(|_req| async {
+            Ok(ControlFlow::Break(
+                ExecutionResponse::fake_builder()
+                    .label("returned_before_mock_service".to_string())
+                    .build()
+                    .unwrap(),
+            ))
+        })
+        .layer(router_service);
+
+        let request = ExecutionRequest::fake_builder().build();
+
+        let actual_label = service_stack
+            .oneshot(request)
+            .await
+            .unwrap()
+            .next_response()
+            .await
+            .unwrap()
+            .label
+            .unwrap();
+
+        assert_eq!(actual_label, expected_label)
+    }
+
+    #[tokio::test]
+    async fn test_error_oneshot() {
+        let expected_error = "checkpoint_error";
+        let router_service = MockExecutionService::new();
+
+        let service_stack = OneShotAsyncCheckpointLayer::new(move |_req| async move {
+            Err(BoxError::from(expected_error))
+        })
+        .layer(router_service);
+
+        let request = ExecutionRequest::fake_builder().build();
+
+        let actual_error = service_stack
+            .oneshot(request)
+            .await
+            .map(|_| unreachable!())
+            .unwrap_err()
+            .to_string();
+
+        assert_eq!(actual_error, expected_error)
+    }
 }
diff --git a/apollo-router/src/layers/mod.rs b/apollo-router/src/layers/mod.rs
index 14e10205c1..8a175639c9 100644
--- a/apollo-router/src/layers/mod.rs
+++ b/apollo-router/src/layers/mod.rs
@@ -14,6 +14,7 @@ use self::map_first_graphql_response::MapFirstGraphqlResponseLayer;
 use self::map_first_graphql_response::MapFirstGraphqlResponseService;
 use crate::graphql;
 use crate::layers::async_checkpoint::AsyncCheckpointLayer;
+use crate::layers::async_checkpoint::OneShotAsyncCheckpointLayer;
 use crate::layers::instrument::InstrumentLayer;
 use crate::layers::map_future_with_request_data::MapFutureWithRequestDataLayer;
 use crate::layers::map_future_with_request_data::MapFutureWithRequestDataService;
@@ -147,6 +148,63 @@ pub trait ServiceBuilderExt: Sized {
         self.layer(AsyncCheckpointLayer::new(async_checkpoint_fn))
     }
 
+    /// Decide if processing should continue or not, and if not allow returning of a response.
+    /// Unlike checkpoint it is possible to perform async operations in the callback. Unlike
+    /// checkpoint_async, this does not require that the service is `Clone` and avoids the
+    /// requiremnent to buffer services.
+    ///
+    /// This is useful for things like authentication where you need to make an external call to
+    /// check if a request should proceed or not.
+    ///
+    /// # Arguments
+    ///
+    /// * `async_checkpoint_fn`: The asynchronous callback to decide if processing should continue or not.
+    ///
+    /// returns: ServiceBuilder, L>>
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// # use std::ops::ControlFlow;
+    /// use futures::FutureExt;
+    /// # use http::Method;
+    /// # use tower::ServiceBuilder;
+    /// # use tower_service::Service;
+    /// # use tracing::info_span;
+    /// # use apollo_router::services::supergraph;
+    /// # use apollo_router::layers::ServiceBuilderExt;
+    /// # fn test(service: supergraph::BoxService) {
+    /// let _ = ServiceBuilder::new()
+    ///     .oneshot_checkpoint_async(|req: supergraph::Request|
+    ///         async {
+    ///             if req.supergraph_request.method() == Method::GET {
+    ///                 Ok(ControlFlow::Break(supergraph::Response::builder()
+    ///                     .data("Only get requests allowed")
+    ///                     .context(req.context)
+    ///                     .build()?))
+    ///             } else {
+    ///                 Ok(ControlFlow::Continue(req))
+    ///             }
+    ///         }
+    ///         .boxed()
+    ///     )
+    ///     .service(service);
+    /// # }
+    /// ```
+    fn oneshot_checkpoint_async(
+        self,
+        async_checkpoint_fn: F,
+    ) -> ServiceBuilder, L>>
+    where
+        S: Service + Send + 'static,
+        Fut: Future<
+            Output = Result>::Response, Request>, BoxError>,
+        >,
+        F: Fn(Request) -> Fut + Send + Sync + 'static,
+    {
+        self.layer(OneShotAsyncCheckpointLayer::new(async_checkpoint_fn))
+    }
+
     /// Adds a buffer to the service stack with a default size.
     ///
     /// This is useful for making services `Clone` and `Send`
diff --git a/apollo-router/src/plugins/coprocessor/mod.rs b/apollo-router/src/plugins/coprocessor/mod.rs
index 5580c3d2da..91b8a087de 100644
--- a/apollo-router/src/plugins/coprocessor/mod.rs
+++ b/apollo-router/src/plugins/coprocessor/mod.rs
@@ -31,7 +31,7 @@ use tower::ServiceBuilder;
 use tower::ServiceExt;
 
 use crate::error::Error;
-use crate::layers::async_checkpoint::AsyncCheckpointLayer;
+use crate::layers::async_checkpoint::OneShotAsyncCheckpointLayer;
 use crate::layers::ServiceBuilderExt;
 use crate::plugin::Plugin;
 use crate::plugin::PluginInit;
@@ -308,7 +308,7 @@ impl RouterStage {
             let http_client = http_client.clone();
             let sdl = sdl.clone();
 
-            AsyncCheckpointLayer::new(move |request: router::Request| {
+            OneShotAsyncCheckpointLayer::new(move |request: router::Request| {
                 let request_config = request_config.clone();
                 let coprocessor_url = coprocessor_url.clone();
                 let http_client = http_client.clone();
@@ -394,7 +394,6 @@ impl RouterStage {
             .instrument(external_service_span())
             .option_layer(request_layer)
             .option_layer(response_layer)
-            .buffered()
             .service(service)
             .boxed()
     }
@@ -441,7 +440,7 @@ impl SubgraphStage {
             let http_client = http_client.clone();
             let coprocessor_url = coprocessor_url.clone();
             let service_name = service_name.clone();
-            AsyncCheckpointLayer::new(move |request: subgraph::Request| {
+            OneShotAsyncCheckpointLayer::new(move |request: subgraph::Request| {
                 let http_client = http_client.clone();
                 let coprocessor_url = coprocessor_url.clone();
                 let service_name = service_name.clone();
@@ -528,7 +527,6 @@ impl SubgraphStage {
             .instrument(external_service_span())
             .option_layer(request_layer)
             .option_layer(response_layer)
-            .buffered()
             .service(service)
             .boxed()
     }
diff --git a/apollo-router/src/plugins/coprocessor/supergraph.rs b/apollo-router/src/plugins/coprocessor/supergraph.rs
index a76de0763e..389207153b 100644
--- a/apollo-router/src/plugins/coprocessor/supergraph.rs
+++ b/apollo-router/src/plugins/coprocessor/supergraph.rs
@@ -13,7 +13,7 @@ use tower_service::Service;
 use super::externalize_header_map;
 use super::*;
 use crate::graphql;
-use crate::layers::async_checkpoint::AsyncCheckpointLayer;
+use crate::layers::async_checkpoint::OneShotAsyncCheckpointLayer;
 use crate::layers::ServiceBuilderExt;
 use crate::plugins::coprocessor::EXTERNAL_SPAN_NAME;
 use crate::response;
@@ -82,7 +82,7 @@ impl SupergraphStage {
             let http_client = http_client.clone();
             let sdl = sdl.clone();
 
-            AsyncCheckpointLayer::new(move |request: supergraph::Request| {
+            OneShotAsyncCheckpointLayer::new(move |request: supergraph::Request| {
                 let request_config = request_config.clone();
                 let coprocessor_url = coprocessor_url.clone();
                 let http_client = http_client.clone();
@@ -169,7 +169,6 @@ impl SupergraphStage {
             .instrument(external_service_span())
             .option_layer(request_layer)
             .option_layer(response_layer)
-            .buffered()
             .service(service)
             .boxed()
     }
diff --git a/apollo-router/src/services/layers/allow_only_http_post_mutations.rs b/apollo-router/src/services/layers/allow_only_http_post_mutations.rs
index 29a58fd2ef..ed1ad14fc1 100644
--- a/apollo-router/src/services/layers/allow_only_http_post_mutations.rs
+++ b/apollo-router/src/services/layers/allow_only_http_post_mutations.rs
@@ -20,7 +20,7 @@ use tower::ServiceBuilder;
 use super::query_analysis::Compiler;
 use crate::graphql::Error;
 use crate::json_ext::Object;
-use crate::layers::async_checkpoint::AsyncCheckpointService;
+use crate::layers::async_checkpoint::OneShotAsyncCheckpointService;
 use crate::layers::ServiceBuilderExt;
 use crate::services::SupergraphRequest;
 use crate::services::SupergraphResponse;
@@ -37,7 +37,7 @@ where
         + 'static,
     >::Future: Send + 'static,
 {
-    type Service = AsyncCheckpointService<
+    type Service = OneShotAsyncCheckpointService<
         S,
         BoxFuture<'static, Result, BoxError>>,
         SupergraphRequest,
@@ -45,7 +45,7 @@ where
 
     fn layer(&self, service: S) -> Self::Service {
         ServiceBuilder::new()
-            .checkpoint_async(|req: SupergraphRequest| {
+            .oneshot_checkpoint_async(|req: SupergraphRequest| {
                 Box::pin(async {
                     if req.supergraph_request.method() == Method::POST {
                         return Ok(ControlFlow::Continue(req));
@@ -148,15 +148,10 @@ mod forbid_http_get_mutations_tests {
     async fn it_lets_http_post_queries_pass_through() {
         let mut mock_service = MockSupergraphService::new();
 
-        mock_service.expect_clone().returning(move || {
-            let mut service = MockSupergraphService::new();
-
-            service
-                .expect_call()
-                .times(1)
-                .returning(move |_| Ok(SupergraphResponse::fake_builder().build().unwrap()));
-            service
-        });
+        mock_service
+            .expect_call()
+            .times(1)
+            .returning(move |_| Ok(SupergraphResponse::fake_builder().build().unwrap()));
 
         let mut service_stack = AllowOnlyHttpPostMutationsLayer::default().layer(mock_service);
 
@@ -176,15 +171,10 @@ mod forbid_http_get_mutations_tests {
     async fn it_lets_http_post_mutations_pass_through() {
         let mut mock_service = MockSupergraphService::new();
 
-        mock_service.expect_clone().returning(move || {
-            let mut service = MockSupergraphService::new();
-
-            service
-                .expect_call()
-                .times(1)
-                .returning(move |_| Ok(SupergraphResponse::fake_builder().build().unwrap()));
-            service
-        });
+        mock_service
+            .expect_call()
+            .times(1)
+            .returning(move |_| Ok(SupergraphResponse::fake_builder().build().unwrap()));
 
         let mut service_stack = AllowOnlyHttpPostMutationsLayer::default().layer(mock_service);
 
@@ -204,15 +194,10 @@ mod forbid_http_get_mutations_tests {
     async fn it_lets_http_get_queries_pass_through() {
         let mut mock_service = MockSupergraphService::new();
 
-        mock_service.expect_clone().returning(move || {
-            let mut service = MockSupergraphService::new();
-
-            service
-                .expect_call()
-                .times(1)
-                .returning(move |_| Ok(SupergraphResponse::fake_builder().build().unwrap()));
-            service
-        });
+        mock_service
+            .expect_call()
+            .times(1)
+            .returning(move |_| Ok(SupergraphResponse::fake_builder().build().unwrap()));
 
         let mut service_stack = AllowOnlyHttpPostMutationsLayer::default().layer(mock_service);
 
@@ -244,12 +229,6 @@ mod forbid_http_get_mutations_tests {
         let expected_status = StatusCode::METHOD_NOT_ALLOWED;
         let expected_allow_header = "POST";
 
-        let mut mock_service = MockSupergraphService::new();
-        mock_service
-            .expect_clone()
-            .returning(MockSupergraphService::new);
-        let mut service_stack = AllowOnlyHttpPostMutationsLayer::default().layer(mock_service);
-
         let forbidden_requests = [
             Method::GET,
             Method::HEAD,
@@ -263,9 +242,11 @@ mod forbid_http_get_mutations_tests {
         .into_iter()
         .map(|method| create_request(method, OperationKind::Mutation));
 
-        let services = service_stack.ready().await.unwrap();
-
         for request in forbidden_requests {
+            let mock_service = MockSupergraphService::new();
+            let mut service_stack = AllowOnlyHttpPostMutationsLayer::default().layer(mock_service);
+            let services = service_stack.ready().await.unwrap();
+
             let mut actual_error = services.call(request).await.unwrap();
 
             assert_eq!(expected_status, actual_error.response.status());
diff --git a/docs/source/customizations/native.mdx b/docs/source/customizations/native.mdx
index 3ad95c7cb9..db8d3d4aa8 100644
--- a/docs/source/customizations/native.mdx
+++ b/docs/source/customizations/native.mdx
@@ -164,6 +164,7 @@ Some notable layers are:
 * **buffered** - Make a service `Clone`. Typically required for any `async` layers.
 * **checkpoint** - Perform a sync call to decide if a request should proceed or not. Useful for validation.
 * **checkpoint_async** - Perform an async call to decide if the request should proceed or not. e.g. for Authentication. Requires `buffered`.
+* **oneshot_checkpoint_async** - Perform an async call to decide if the request should proceed or not. e.g. for Authentication. Does not require `buffered` and should be preferred to `checkpoint_async` for that reason.
 * **instrument** - Add a tracing span around a service.
 * **map_request** - Transform the request before proceeding. e.g. for header manipulation.
 * **map_response** - Transform the response before proceeding. e.g. for header manipulation.
diff --git a/examples/async-auth/rust/src/allow_client_id_from_file.rs b/examples/async-auth/rust/src/allow_client_id_from_file.rs
index a62fbc07a5..a41e6ce370 100644
--- a/examples/async-auth/rust/src/allow_client_id_from_file.rs
+++ b/examples/async-auth/rust/src/allow_client_id_from_file.rs
@@ -48,12 +48,12 @@ impl Plugin for AllowClientIdFromFile {
     // switching the async file read with an async http request
     fn supergraph_service(&self, service: supergraph::BoxService) -> supergraph::BoxService {
         let header_key = self.header.clone();
-        // async_checkpoint is an async function.
+        // oneshot_async_checkpoint is an async function.
         // this means it will run whenever the service `await`s it
         // given we're getting a mutable reference to self,
         // self won't be present anymore when we `await` the checkpoint.
         //
-        // this is solved by cloning the path and moving it into the async_checkpoint callback.
+        // this is solved by cloning the path and moving it into the oneshot_async_checkpoint callback.
         //
         // see https://rust-lang.github.io/async-book/03_async_await/01_chapter.html#async-lifetimes for more information
         let allowed_ids_path = self.allowed_ids_path.clone();
@@ -141,20 +141,12 @@ impl Plugin for AllowClientIdFromFile {
                 }
             }
         };
-        // `ServiceBuilder` provides us with an `async_checkpoint` method.
+        // `ServiceBuilder` provides us with an `oneshot_async_checkpoint` method.
         //
         // This method allows us to return ControlFlow::Continue(request) if we want to let the request through,
         // or ControlFlow::Break(response) with a crafted response if we don't want the request to go through.
         ServiceBuilder::new()
-            .checkpoint_async(handler)
-            // Given the async nature of our checkpoint, we need to make sure
-            // the underlying service will be available whenever the checkpoint
-            // returns ControlFlow::Continue.
-            // This is achieved by adding a buffer in front of the service,
-            // and (automatically) giving one `slot` to our async_checkpoint
-            //
-            // forgetting to add .buffer() here will trigger a compilation error.
-            .buffer(20_000)
+            .oneshot_checkpoint_async(handler)
             .service(service)
             .boxed()
     }

From 765cf39612cd718f0f1457e087ed41798997ce84 Mon Sep 17 00:00:00 2001
From: Geoffroy Couprie 
Date: Fri, 22 Sep 2023 10:25:36 +0200
Subject: [PATCH 34/51] document the query plan cache warm up (#3815)

Fix #3704

Co-authored-by: Bryn Cooke 
Co-authored-by: Edward Huang <18322228+shorgi@users.noreply.github.com>
---
 .../feat_geal_plan_cache_warmup_doc.md        | 13 +++++++
 .../configuration/in-memory-caching.mdx       | 39 +++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 .changesets/feat_geal_plan_cache_warmup_doc.md

diff --git a/.changesets/feat_geal_plan_cache_warmup_doc.md b/.changesets/feat_geal_plan_cache_warmup_doc.md
new file mode 100644
index 0000000000..a7c9a6f2f1
--- /dev/null
+++ b/.changesets/feat_geal_plan_cache_warmup_doc.md
@@ -0,0 +1,13 @@
+### Query plan cache warm-up improvements ([Issue #3704](https://github.com/apollographql/router/issues/3704))
+
+The `warm_up_queries` option enables quicker schema updates by precomputing query plans for your most used cached queries and your persisted queries. When a new schema is loaded, a precomputed query plan for it may already be in in-memory cache.
+
+We made a series of improvements to this feature to make it more usable:
+* It is now active by default and warms up the cache with the 30% most used queries of the previous cache. The amount is still configurable, and it can be deactivated by setting it to 0.
+* We added new metrics to track the time spent loading a new schema and planning queries in the warm-up phase. You can also measure the query plan cache usage, to know how many entries are used, and the cache hit rate, both for the in memory cache and the distributed cache.
+* The warm-up will now plan queries in random order, to make sure that the work can be shared by multiple router instances using distributed caching
+* Persisted queries are part of the warmed up queries.
+
+You can get more information about operating the query plan cache and its warm-up phase in the [documentation](https://www.apollographql.com/docs/router/configuration/in-memory-caching#cache-warm-up)
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3815 https://github.com/apollographql/router/pull/3801 https://github.com/apollographql/router/pull/3767 https://github.com/apollographql/router/pull/3769 https://github.com/apollographql/router/pull/3770
\ No newline at end of file
diff --git a/docs/source/configuration/in-memory-caching.mdx b/docs/source/configuration/in-memory-caching.mdx
index fe8de83bcc..6606caee60 100644
--- a/docs/source/configuration/in-memory-caching.mdx
+++ b/docs/source/configuration/in-memory-caching.mdx
@@ -42,6 +42,45 @@ supergraph:
         limit: 512
 ```
 
+### Cache warm-up
+
+When loading a new schema, a query plan might change for some queries, so cached query plans cannot be reused. 
+
+To prevent increased latency upon query plan cache invalidation, the Router precomputes query plans for:
+* The most used queries from the cache.
+* The entire list of persisted queries.
+
+Precomputed plans will be cached before the Router switches traffic over to the new schema.
+
+By default, the Router warms up the cache with 30% of the queries already in cache, but it can be configured as follows:
+
+```yaml title="router.yaml"
+supergraph:
+  query_planning:
+    # Pre-plan the 100 most used operations when the supergraph changes
+    warmed_up_queries: 100
+```
+
+To get more information on the planning and warm-up process use the following metrics:
+
+(`` can be `redis` for distributed cache or `memory`)
+* counters:
+  * `apollo_router_cache_size{kind="query planner", storage="}`: current size of the cache (only for in-memory cache)
+  * `apollo_router_cache_hit_count{kind="query planner", storage="}`
+  * `apollo_router_cache_miss_count{kind="query planner", storage="}`
+
+* histograms:
+  * `apollo_router_query_planning_time`: time spent planning queries
+  * `apollo_router_schema_loading_time`: time spent loading a schema
+  * `apollo_router_cache_hit_time{kind="query planner", storage="}`: time to get a value from the cache
+  * `apollo_router_cache_miss_time{kind="query planner", storage="}`
+
+Typically, we would look at `apollo_router_cache_size` and the cache hit rate to define the right size of the in memory cache,
+then look at `apollo_router_schema_loading_time` and `apollo_router_query_planning_time` to decide how much time we want to spend warming up queries.
+
+#### Cache warm-up with distributed caching
+
+If the Router is using distributed caching for query plans, the warm-up phase will also store the new query plans in Redis. Since all Router instances might have the same distributions of queries in their in-memory cache, the list of queries is shuffled before warm-up, so each Router instance can plan queries in a different order and share their results through the cache.
 ## Caching automatic persisted queries (APQ)
 
 **Automatic Persisted Queries** (**APQ**) enable GraphQL clients to send a server the _hash_ of their query string, _instead of_ sending the query string itself. When query strings are very large, this can significantly reduce network usage.

From dfed65001f274a0720c73a2324320bd33f0fbba5 Mon Sep 17 00:00:00 2001
From: Jeremy Lempereur 
Date: Fri, 22 Sep 2023 12:30:25 +0200
Subject: [PATCH 35/51] Don't panic in OneShotAsyncCheckpoint. (#3878)

Followup to #3819, this removes a panic which would trigger if a user
misuses OneShotAsyncCheckpoint.
---
 apollo-router/src/layers/async_checkpoint.rs | 60 ++++++++++++++++++--
 1 file changed, 54 insertions(+), 6 deletions(-)

diff --git a/apollo-router/src/layers/async_checkpoint.rs b/apollo-router/src/layers/async_checkpoint.rs
index a05bb6a78a..5cd45e1713 100644
--- a/apollo-router/src/layers/async_checkpoint.rs
+++ b/apollo-router/src/layers/async_checkpoint.rs
@@ -30,7 +30,7 @@ where
     Fut: Future>::Response, Request>, BoxError>>,
 {
     checkpoint_fn: Arc Fut + Send + Sync + 'static>>>,
-    phantom: PhantomData, // XXX: The compiler can't detect that S is used in the Future...
+    phantom: PhantomData, // We use PhantomData because the compiler can't detect that S is used in the Future.
 }
 
 impl AsyncCheckpointLayer
@@ -131,11 +131,9 @@ where
 
     fn call(&mut self, req: Request) -> Self::Future {
         let checkpoint_fn = Arc::clone(&self.checkpoint_fn);
-        let inner = self
-            .inner
-            .take()
-            .expect("One shot must only be called once");
+        let inner = self.inner.take();
         Box::pin(async move {
+            let inner = inner.ok_or("One shot must only be called once")?;
             match (checkpoint_fn)(req).await {
                 Ok(ControlFlow::Break(response)) => Ok(response),
                 Ok(ControlFlow::Continue(request)) => inner.oneshot(request).await,
@@ -223,7 +221,7 @@ where
     Fut: Future>::Response, Request>, BoxError>>,
 {
     checkpoint_fn: Arc Fut + Send + Sync + 'static>>>,
-    phantom: PhantomData, // XXX: The compiler can't detect that S is used in the Future...
+    phantom: PhantomData, // We use PhantomData because the compiler can't detect that S is used in the Future.
 }
 
 impl OneShotAsyncCheckpointLayer
@@ -564,4 +562,54 @@ mod async_checkpoint_tests {
 
         assert_eq!(actual_error, expected_error)
     }
+
+    #[tokio::test]
+    async fn test_double_ready_doesnt_panic() {
+        let router_service = MockExecutionService::new();
+
+        let mut service_stack = OneShotAsyncCheckpointLayer::new(|_req| async {
+            Ok(ControlFlow::Break(
+                ExecutionResponse::fake_builder()
+                    .label("returned_before_mock_service".to_string())
+                    .build()
+                    .unwrap(),
+            ))
+        })
+        .layer(router_service);
+
+        service_stack.ready().await.unwrap();
+        service_stack
+            .call(ExecutionRequest::fake_builder().build())
+            .await
+            .unwrap();
+
+        assert!(service_stack.ready().await.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_double_call_doesnt_panic() {
+        let router_service = MockExecutionService::new();
+
+        let mut service_stack = OneShotAsyncCheckpointLayer::new(|_req| async {
+            Ok(ControlFlow::Break(
+                ExecutionResponse::fake_builder()
+                    .label("returned_before_mock_service".to_string())
+                    .build()
+                    .unwrap(),
+            ))
+        })
+        .layer(router_service);
+
+        service_stack.ready().await.unwrap();
+
+        service_stack
+            .call(ExecutionRequest::fake_builder().build())
+            .await
+            .unwrap();
+
+        assert!(service_stack
+            .call(ExecutionRequest::fake_builder().build())
+            .await
+            .is_err());
+    }
 }

From aa85b15f6f775d9757016afe7e87d73294fd26c1 Mon Sep 17 00:00:00 2001
From: Bryn Cooke 
Date: Fri, 22 Sep 2023 14:06:38 +0100
Subject: [PATCH 36/51] preverify release version (#3859)

Adds a new `xtask release pre-verify`

Adds a check that the cargo toml matches the local tagged tree.

Fixes #3664


---

**Checklist**

Complete the checklist (and note appropriate exceptions) before the PR
is marked ready-for-review.

- [ ] Changes are compatible[^1]
- [ ] Documentation[^2] completed
- [ ] Performance impact assessed and acceptable
- Tests added and passing[^3]
    - [ ] Unit Tests
    - [ ] Integration Tests
    - [ ] Manual Tests

**Exceptions**

*Note any exceptions here*

**Notes**

[^1]: It may be appropriate to bring upcoming changes to the attention
of other (impacted) groups. Please endeavour to do this before seeking
PR approval. The mechanism for doing this will vary considerably, so use
your judgement as to how and when to do this.
[^2]: Configuration is an important part of many changes. Where
applicable please try to document configuration examples.
[^3]: Tick whichever testing boxes are applicable. If you are adding
Manual Tests, please document the manual testing (extensively) in the
Exceptions.

---------

Co-authored-by: bryn 
---
 .circleci/config.yml          | 24 +++++++++++++++++++
 xtask/src/commands/release.rs | 44 +++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index bd7fa34a99..c7db107850 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -378,6 +378,9 @@ commands:
                 key: "<< pipeline.parameters.merge_version >>-lint"
                 paths:
                   - target
+  xtask_release_preverify:
+    steps:
+      - run: xtask release pre-verify
 
   xtask_check_helm:
     steps:
@@ -529,6 +532,18 @@ jobs:
             cargo fetch
       - xtask_test:
           variant: "updated"
+  pre_verify_release:
+    environment:
+      <<: *common_job_environment
+    parameters:
+      platform:
+        type: executor
+    executor: << parameters.platform >>
+    steps:
+      - checkout
+      - setup_environment:
+          platform: << parameters.platform >>
+      - xtask_release_preverify
 
   build_release:
     parameters:
@@ -843,6 +858,15 @@ workflows:
     when:
       not: << pipeline.parameters.nightly >>
     jobs:
+      - pre_verify_release:
+          matrix:
+            parameters:
+              platform: [ amd_linux_build ]
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /v.*/
       - build_release:
           matrix:
             parameters:
diff --git a/xtask/src/commands/release.rs b/xtask/src/commands/release.rs
index a11f9159c6..7375753f33 100644
--- a/xtask/src/commands/release.rs
+++ b/xtask/src/commands/release.rs
@@ -14,12 +14,16 @@ use crate::commands::changeset::slurp_and_remove_changesets;
 pub enum Command {
     /// Prepare a new release
     Prepare(Prepare),
+
+    /// Verify that a release is ready to be published
+    PreVerify,
 }
 
 impl Command {
     pub fn run(&self) -> Result<()> {
         match self {
             Command::Prepare(command) => command.run(),
+            Command::PreVerify => PreVerify::run(),
         }
     }
 }
@@ -410,3 +414,43 @@ impl Prepare {
         Ok(())
     }
 }
+
+struct PreVerify();
+
+impl PreVerify {
+    fn run() -> Result<()> {
+        let version = format!("v{}", *PKG_VERSION);
+
+        // Get the git tag name as a string
+        let tags_output = std::process::Command::new("git")
+            .args(["describe", "--tags", "--exact-match"])
+            .output()
+            .map_err(|e| {
+                anyhow!(
+                    "failed to execute 'git describe --tags --exact-match': {}",
+                    e
+                )
+            })?
+            .stdout;
+        let tags_raw = String::from_utf8_lossy(&tags_output);
+        let tags_list = tags_raw
+            .split("\n")
+            .filter(|s| !s.trim().is_empty())
+            .collect::>();
+
+        // If the tags contains the version, then we're good
+        if tags_list.is_empty() {
+            return Err(anyhow!(
+                "release cannot be performed because current git tree is not tagged"
+            ));
+        }
+        if !tags_list.contains(&version.as_str()) {
+            return Err(anyhow!(
+                "the git tree tags {{{}}} does not contain the version {} from the Cargo.toml",
+                tags_list.join(", "),
+                version
+            ));
+        }
+        Ok(())
+    }
+}

From 3f46814f2430d15cdf469a817f6226526d0fa739 Mon Sep 17 00:00:00 2001
From: Geoffroy Couprie 
Date: Fri, 22 Sep 2023 17:07:47 +0200
Subject: [PATCH 37/51] fewer header manipulations and allocations (#3844)

`HeaderName::from_static` and `HeaderValue::from_static` can avoid
allocations, but we do not get the full benefit from this function by
calling them directly in the code: They are const functions, so their
result should be stored in a const or static value, then reused
elsewhere.
This also removes some unneeded header allocations when handling client
name and client version
---
 apollo-router/src/http_ext.rs                 |  9 ++-
 apollo-router/src/plugins/telemetry/mod.rs    | 63 +++++++++----------
 .../telemetry/tracing/apollo_telemetry.rs     |  2 +-
 apollo-router/src/services/layers/apq.rs      |  6 +-
 .../services/layers/content_negotiation.rs    | 27 ++++----
 apollo-router/src/services/router.rs          |  8 +--
 apollo-router/src/services/router_service.rs  | 36 ++++++-----
 .../src/services/subgraph_service.rs          | 23 ++++---
 8 files changed, 92 insertions(+), 82 deletions(-)

diff --git a/apollo-router/src/http_ext.rs b/apollo-router/src/http_ext.rs
index be1b4554ae..b30982ec1c 100644
--- a/apollo-router/src/http_ext.rs
+++ b/apollo-router/src/http_ext.rs
@@ -13,10 +13,10 @@ use bytes::Bytes;
 use http::header;
 use http::header::HeaderName;
 use http::HeaderValue;
-use mime::APPLICATION_JSON;
 use multimap::MultiMap;
 
 use crate::graphql;
+use crate::services::APPLICATION_JSON_HEADER_VALUE;
 
 /// Delayed-fallibility wrapper for conversion to [`http::header::HeaderName`].
 ///
@@ -441,10 +441,9 @@ impl IntoResponse for Response {
         let (mut parts, body) = http::Response::from(self).into_parts();
         let json_body_bytes =
             Bytes::from(serde_json::to_vec(&body).expect("body should be serializable; qed"));
-        parts.headers.insert(
-            header::CONTENT_TYPE,
-            HeaderValue::from_static(APPLICATION_JSON.essence_str()),
-        );
+        parts
+            .headers
+            .insert(header::CONTENT_TYPE, APPLICATION_JSON_HEADER_VALUE.clone());
 
         axum::response::Response::from_parts(parts, boxed(http_body::Full::new(json_body_bytes)))
     }
diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs
index 82da7522a7..976c7a6129 100644
--- a/apollo-router/src/plugins/telemetry/mod.rs
+++ b/apollo-router/src/plugins/telemetry/mod.rs
@@ -151,6 +151,10 @@ pub(crate) const LOGGING_DISPLAY_BODY: &str = "apollo_telemetry::logging::displa
 const DEFAULT_SERVICE_NAME: &str = "apollo-router";
 const GLOBAL_TRACER_NAME: &str = "apollo-router";
 const DEFAULT_EXPOSE_TRACE_ID_HEADER: &str = "apollo-trace-id";
+static DEFAULT_EXPOSE_TRACE_ID_HEADER_NAME: HeaderName =
+    HeaderName::from_static(DEFAULT_EXPOSE_TRACE_ID_HEADER);
+static FTV1_HEADER_NAME: HeaderName = HeaderName::from_static("apollo-federation-include-trace");
+static FTV1_HEADER_VALUE: HeaderValue = HeaderValue::from_static("ftv1");
 
 #[doc(hidden)] // Only public for integration tests
 pub(crate) struct Telemetry {
@@ -315,21 +319,21 @@ impl Plugin for Telemetry {
                     .unwrap_or_default();
                 let router_request = &request.router_request;
                 let headers = router_request.headers();
-                let client_name = headers
+                let client_name: &str = headers
                     .get(&apollo.client_name_header)
-                    .cloned()
-                    .unwrap_or_else(|| HeaderValue::from_static(""));
+                    .and_then(|h| h.to_str().ok())
+                    .unwrap_or("");
                 let client_version = headers
                     .get(&apollo.client_version_header)
-                    .cloned()
-                    .unwrap_or_else(|| HeaderValue::from_static(""));
+                    .and_then(|h| h.to_str().ok())
+                    .unwrap_or("");
                 let span = ::tracing::info_span!(ROUTER_SPAN_NAME,
                     "http.method" = %router_request.method(),
                     "http.route" = %router_request.uri(),
                     "http.flavor" = ?router_request.version(),
                     "trace_id" = %trace_id,
-                    "client.name" = client_name.to_str().unwrap_or_default(),
-                    "client.version" = client_version.to_str().unwrap_or_default(),
+                    "client.name" = client_name,
+                    "client.version" = client_version,
                     "otel.kind" = "INTERNAL",
                     "otel.status_code" = ::tracing::field::Empty,
                     "apollo_private.duration_ns" = ::tracing::field::Empty,
@@ -411,7 +415,7 @@ impl Plugin for Telemetry {
                         t.response_trace_id
                             .header_name
                             .clone()
-                            .unwrap_or(HeaderName::from_static(DEFAULT_EXPOSE_TRACE_ID_HEADER))
+                            .unwrap_or_else(||DEFAULT_EXPOSE_TRACE_ID_HEADER_NAME.clone())
                     })
                 });
                 if let (Some(header_name), Some(trace_id)) = (
@@ -934,26 +938,22 @@ impl Telemetry {
         let headers = http_request.headers();
         let client_name_header = &apollo_config.client_name_header;
         let client_version_header = &apollo_config.client_version_header;
-        let _ = context.insert(
-            CLIENT_NAME,
-            headers
-                .get(client_name_header)
-                .cloned()
-                .unwrap_or_else(|| HeaderValue::from_static(""))
-                .to_str()
-                .unwrap_or_default()
-                .to_string(),
-        );
-        let _ = context.insert(
-            CLIENT_VERSION,
-            headers
-                .get(client_version_header)
-                .cloned()
-                .unwrap_or_else(|| HeaderValue::from_static(""))
-                .to_str()
-                .unwrap_or_default()
-                .to_string(),
-        );
+        if let Some(name) = headers
+            .get(client_name_header)
+            .and_then(|h| h.to_str().ok())
+            .map(|s| s.to_owned())
+        {
+            let _ = context.insert(CLIENT_NAME, name);
+        }
+
+        if let Some(version) = headers
+            .get(client_version_header)
+            .and_then(|h| h.to_str().ok())
+            .map(|s| s.to_owned())
+        {
+            let _ = context.insert(CLIENT_VERSION, version);
+        }
+
         let (should_log_headers, should_log_body) = config.logging.should_log(req);
         if should_log_headers {
             ::tracing::info!(http.request.headers = ?req.supergraph_request.headers(), "Supergraph request headers");
@@ -1888,10 +1888,9 @@ fn request_ftv1(mut req: SubgraphRequest) -> SubgraphRequest {
         .contains_key::()
         && Span::current().context().span().span_context().is_sampled()
     {
-        req.subgraph_request.headers_mut().insert(
-            "apollo-federation-include-trace",
-            HeaderValue::from_static("ftv1"),
-        );
+        req.subgraph_request
+            .headers_mut()
+            .insert(FTV1_HEADER_NAME.clone(), FTV1_HEADER_VALUE.clone());
     }
     req
 }
diff --git a/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs b/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs
index 25f260977c..986a119113 100644
--- a/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs
+++ b/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs
@@ -382,7 +382,7 @@ impl Exporter {
             SUBGRAPH_SPAN_NAME => {
                 let subgraph_name = span
                     .attributes
-                    .get(&Key::from_static_str("apollo.subgraph.name"))
+                    .get(&SUBGRAPH_NAME)
                     .and_then(extract_string)
                     .unwrap_or_default();
                 let error_configuration = self
diff --git a/apollo-router/src/services/layers/apq.rs b/apollo-router/src/services/layers/apq.rs
index 2b27b817b2..296896199f 100644
--- a/apollo-router/src/services/layers/apq.rs
+++ b/apollo-router/src/services/layers/apq.rs
@@ -17,6 +17,7 @@ use crate::services::SupergraphRequest;
 use crate::services::SupergraphResponse;
 
 const DONT_CACHE_RESPONSE_VALUE: &str = "private, no-cache, must-revalidate";
+static DONT_CACHE_HEADER_VALUE: HeaderValue = HeaderValue::from_static(DONT_CACHE_RESPONSE_VALUE);
 
 /// A persisted query.
 #[derive(Deserialize, Clone, Debug)]
@@ -134,10 +135,7 @@ async fn apq_request(
                     // Persisted query errors (especially "not found") need to be uncached, because
                     // hopefully we're about to fill in the APQ cache and the same request will
                     // succeed next time.
-                    .header(
-                        CACHE_CONTROL,
-                        HeaderValue::from_static(DONT_CACHE_RESPONSE_VALUE),
-                    )
+                    .header(CACHE_CONTROL, DONT_CACHE_HEADER_VALUE.clone())
                     .context(request.context)
                     .build()
                     .expect("response is valid");
diff --git a/apollo-router/src/services/layers/content_negotiation.rs b/apollo-router/src/services/layers/content_negotiation.rs
index ef49a93416..42d9a30c40 100644
--- a/apollo-router/src/services/layers/content_negotiation.rs
+++ b/apollo-router/src/services/layers/content_negotiation.rs
@@ -3,7 +3,6 @@ use std::ops::ControlFlow;
 use http::header::ACCEPT;
 use http::header::CONTENT_TYPE;
 use http::HeaderMap;
-use http::HeaderValue;
 use http::Method;
 use http::StatusCode;
 use mediatype::names::APPLICATION;
@@ -24,7 +23,10 @@ use crate::layers::sync_checkpoint::CheckpointService;
 use crate::layers::ServiceExt as _;
 use crate::services::router;
 use crate::services::router::ClientRequestAccepts;
+use crate::services::router_service::MULTIPART_DEFER_HEADER_VALUE;
+use crate::services::router_service::MULTIPART_SUBSCRIPTION_HEADER_VALUE;
 use crate::services::supergraph;
+use crate::services::APPLICATION_JSON_HEADER_VALUE;
 use crate::services::MULTIPART_DEFER_CONTENT_TYPE;
 use crate::services::MULTIPART_DEFER_SPEC_PARAMETER;
 use crate::services::MULTIPART_DEFER_SPEC_VALUE;
@@ -138,20 +140,17 @@ where
                     .unwrap_or_default();
 
                 if !res.has_next.unwrap_or_default() && (accepts_json || accepts_wildcard) {
-                    parts.headers.insert(
-                        CONTENT_TYPE,
-                        HeaderValue::from_static(APPLICATION_JSON.essence_str()),
-                    );
+                    parts
+                        .headers
+                        .insert(CONTENT_TYPE, APPLICATION_JSON_HEADER_VALUE.clone());
                 } else if accepts_multipart_defer {
-                    parts.headers.insert(
-                        CONTENT_TYPE,
-                        HeaderValue::from_static(MULTIPART_DEFER_CONTENT_TYPE),
-                    );
+                    parts
+                        .headers
+                        .insert(CONTENT_TYPE, MULTIPART_DEFER_HEADER_VALUE.clone());
                 } else if accepts_multipart_subscription {
-                    parts.headers.insert(
-                        CONTENT_TYPE,
-                        HeaderValue::from_static(MULTIPART_SUBSCRIPTION_CONTENT_TYPE),
-                    );
+                    parts
+                        .headers
+                        .insert(CONTENT_TYPE, MULTIPART_SUBSCRIPTION_HEADER_VALUE.clone());
                 }
                 (parts, res)
             })
@@ -236,6 +235,8 @@ fn parse_accept(headers: &HeaderMap) -> ClientRequestAccepts {
 
 #[cfg(test)]
 mod tests {
+    use http::HeaderValue;
+
     use super::*;
 
     #[test]
diff --git a/apollo-router/src/services/router.rs b/apollo-router/src/services/router.rs
index 93e735d260..3d51ec2f15 100644
--- a/apollo-router/src/services/router.rs
+++ b/apollo-router/src/services/router.rs
@@ -17,9 +17,9 @@ use serde_json_bytes::Value;
 use static_assertions::assert_impl_all;
 use tower::BoxError;
 
+use super::router_service::MULTIPART_DEFER_HEADER_VALUE;
+use super::router_service::MULTIPART_SUBSCRIPTION_HEADER_VALUE;
 use super::supergraph;
-use super::MULTIPART_DEFER_CONTENT_TYPE;
-use super::MULTIPART_SUBSCRIPTION_CONTENT_TYPE;
 use crate::graphql;
 use crate::json_ext::Path;
 use crate::services::TryIntoHeaderName;
@@ -220,8 +220,8 @@ impl Response {
                 .get(CONTENT_TYPE)
                 .iter()
                 .any(|value| {
-                    *value == HeaderValue::from_static(MULTIPART_DEFER_CONTENT_TYPE)
-                        || *value == HeaderValue::from_static(MULTIPART_SUBSCRIPTION_CONTENT_TYPE)
+                    *value == MULTIPART_DEFER_HEADER_VALUE
+                        || *value == MULTIPART_SUBSCRIPTION_HEADER_VALUE
                 })
             {
                 let multipart = Multipart::new(self.response.into_body(), "graphql");
diff --git a/apollo-router/src/services/router_service.rs b/apollo-router/src/services/router_service.rs
index 8adbf23041..879447c4d4 100644
--- a/apollo-router/src/services/router_service.rs
+++ b/apollo-router/src/services/router_service.rs
@@ -42,6 +42,7 @@ use super::HasPlugins;
 #[cfg(test)]
 use super::HasSchema;
 use super::SupergraphCreator;
+use super::APPLICATION_JSON_HEADER_VALUE;
 use super::MULTIPART_DEFER_CONTENT_TYPE;
 use super::MULTIPART_SUBSCRIPTION_CONTENT_TYPE;
 use crate::cache::DeduplicatingCache;
@@ -63,6 +64,14 @@ use crate::Configuration;
 use crate::Endpoint;
 use crate::ListenAddr;
 
+pub(crate) static MULTIPART_DEFER_HEADER_VALUE: HeaderValue =
+    HeaderValue::from_static(MULTIPART_DEFER_CONTENT_TYPE);
+pub(crate) static MULTIPART_SUBSCRIPTION_HEADER_VALUE: HeaderValue =
+    HeaderValue::from_static(MULTIPART_SUBSCRIPTION_CONTENT_TYPE);
+static ACCEL_BUFFERING_HEADER_NAME: HeaderName = HeaderName::from_static("x-accel-buffering");
+static ACCEL_BUFFERING_HEADER_VALUE: HeaderValue = HeaderValue::from_static("no");
+static ORIGIN_HEADER_VALUE: HeaderValue = HeaderValue::from_static("origin");
+
 /// Containing [`Service`] in the request lifecyle.
 #[derive(Clone)]
 pub(crate) struct RouterService {
@@ -286,10 +295,9 @@ impl RouterService {
                     && !response.subscribed.unwrap_or(false)
                     && (accepts_json || accepts_wildcard)
                 {
-                    parts.headers.insert(
-                        CONTENT_TYPE,
-                        HeaderValue::from_static(APPLICATION_JSON.essence_str()),
-                    );
+                    parts
+                        .headers
+                        .insert(CONTENT_TYPE, APPLICATION_JSON_HEADER_VALUE.clone());
                     tracing::trace_span!("serialize_response").in_scope(|| {
                         let body = serde_json::to_string(&response)?;
                         Ok(router::Response {
@@ -299,20 +307,18 @@ impl RouterService {
                     })
                 } else if accepts_multipart_defer || accepts_multipart_subscription {
                     if accepts_multipart_defer {
-                        parts.headers.insert(
-                            CONTENT_TYPE,
-                            HeaderValue::from_static(MULTIPART_DEFER_CONTENT_TYPE),
-                        );
+                        parts
+                            .headers
+                            .insert(CONTENT_TYPE, MULTIPART_DEFER_HEADER_VALUE.clone());
                     } else if accepts_multipart_subscription {
-                        parts.headers.insert(
-                            CONTENT_TYPE,
-                            HeaderValue::from_static(MULTIPART_SUBSCRIPTION_CONTENT_TYPE),
-                        );
+                        parts
+                            .headers
+                            .insert(CONTENT_TYPE, MULTIPART_SUBSCRIPTION_HEADER_VALUE.clone());
                     }
                     // Useful when you're using a proxy like nginx which enable proxy_buffering by default (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering)
                     parts.headers.insert(
-                        HeaderName::from_static("x-accel-buffering"),
-                        HeaderValue::from_static("no"),
+                        ACCEL_BUFFERING_HEADER_NAME.clone(),
+                        ACCEL_BUFFERING_HEADER_VALUE.clone(),
                     );
                     let multipart_stream = match response.subscribed {
                         Some(true) => {
@@ -456,7 +462,7 @@ impl RouterService {
 fn process_vary_header(headers: &mut HeaderMap) {
     if headers.get(VARY).is_none() {
         // We don't have a VARY header, add one with value "origin"
-        headers.insert(VARY, HeaderValue::from_static("origin"));
+        headers.insert(VARY, ORIGIN_HEADER_VALUE.clone());
     }
 }
 
diff --git a/apollo-router/src/services/subgraph_service.rs b/apollo-router/src/services/subgraph_service.rs
index 9333bc16aa..78e870bf17 100644
--- a/apollo-router/src/services/subgraph_service.rs
+++ b/apollo-router/src/services/subgraph_service.rs
@@ -94,7 +94,10 @@ const POOL_IDLE_TIMEOUT_DURATION: Option = Some(Duration::from_secs(5)
 
 // interior mutability is not a concern here, the value is never modified
 #[allow(clippy::declare_interior_mutable_const)]
-const ACCEPTED_ENCODINGS: HeaderValue = HeaderValue::from_static("gzip, br, deflate");
+static ACCEPTED_ENCODINGS: HeaderValue = HeaderValue::from_static("gzip, br, deflate");
+pub(crate) static APPLICATION_JSON_HEADER_VALUE: HeaderValue =
+    HeaderValue::from_static("application/json");
+static APP_GRAPHQL_JSON: HeaderValue = HeaderValue::from_static(GRAPHQL_JSON_RESPONSE_HEADER_VALUE);
 
 enum APQError {
     PersistedQueryNotSupported,
@@ -704,15 +707,19 @@ async fn call_http(
         })?;
 
     let mut request = http::request::Request::from_parts(parts, compressed_body.into());
-    let app_json: HeaderValue = HeaderValue::from_static(APPLICATION_JSON.essence_str());
-    let app_graphql_json: HeaderValue =
-        HeaderValue::from_static(GRAPHQL_JSON_RESPONSE_HEADER_VALUE);
-    request.headers_mut().insert(CONTENT_TYPE, app_json.clone());
-    request.headers_mut().insert(ACCEPT, app_json);
-    request.headers_mut().append(ACCEPT, app_graphql_json);
+
+    request
+        .headers_mut()
+        .insert(CONTENT_TYPE, APPLICATION_JSON_HEADER_VALUE.clone());
+    request
+        .headers_mut()
+        .insert(ACCEPT, APPLICATION_JSON_HEADER_VALUE.clone());
+    request
+        .headers_mut()
+        .append(ACCEPT, APP_GRAPHQL_JSON.clone());
     request
         .headers_mut()
-        .insert(ACCEPT_ENCODING, ACCEPTED_ENCODINGS);
+        .insert(ACCEPT_ENCODING, ACCEPTED_ENCODINGS.clone());
 
     let schema_uri = request.uri();
     let host = schema_uri.host().unwrap_or_default();

From 0ef2f367230d6b28830e63d34163f37ebb1168f8 Mon Sep 17 00:00:00 2001
From: Geoffroy Couprie 
Date: Fri, 22 Sep 2023 17:08:27 +0200
Subject: [PATCH 38/51] select_object optimizations (#3810)

Small optimizations when selecting entity types and keys for federated
queries

Co-authored-by: o0Ignition0o 
---
 apollo-router/src/query_planner/fetch.rs      | 40 +++++++++----------
 apollo-router/src/query_planner/selection.rs  | 28 +++++++------
 .../src/query_planner/subscription.rs         |  4 +-
 3 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/apollo-router/src/query_planner/fetch.rs b/apollo-router/src/query_planner/fetch.rs
index c7b4e7216f..598fe203f3 100644
--- a/apollo-router/src/query_planner/fetch.rs
+++ b/apollo-router/src/query_planner/fetch.rs
@@ -1,4 +1,3 @@
-use std::collections::HashMap;
 use std::fmt::Display;
 use std::sync::Arc;
 
@@ -98,13 +97,13 @@ pub(crate) struct FetchNode {
 
 pub(crate) struct Variables {
     pub(crate) variables: Object,
-    pub(crate) paths: HashMap,
+    pub(crate) inverted_paths: Vec>,
 }
 
 impl Variables {
     #[instrument(skip_all, level = "debug", name = "make_variables")]
     #[allow(clippy::too_many_arguments)]
-    pub(super) async fn new(
+    pub(super) fn new(
         requires: &[Selection],
         variable_usages: &[String],
         data: &Value,
@@ -123,7 +122,7 @@ impl Variables {
                     .map(|(variable_key, value)| (variable_key.clone(), value.clone()))
             }));
 
-            let mut paths: HashMap = HashMap::new();
+            let mut inverted_paths: Vec> = Vec::new();
             let mut values: IndexSet = IndexSet::new();
 
             data.select_values_and_paths(schema, current_dir, |path, value| {
@@ -132,11 +131,12 @@ impl Variables {
                         rewrites::apply_rewrites(schema, &mut value, input_rewrites);
                         match values.get_index_of(&value) {
                             Some(index) => {
-                                paths.insert(path.clone(), index);
+                                inverted_paths[index].push(path.clone());
                             }
                             None => {
-                                paths.insert(path.clone(), values.len());
+                                inverted_paths.push(vec![path.clone()]);
                                 values.insert(value);
+                                debug_assert!(inverted_paths.len() == values.len());
                             }
                         }
                     }
@@ -151,7 +151,10 @@ impl Variables {
 
             variables.insert("representations", representations);
 
-            Some(Variables { variables, paths })
+            Some(Variables {
+                variables,
+                inverted_paths,
+            })
         } else {
             // with nested operations (Query or Mutation has an operation returning a Query or Mutation),
             // when the first fetch fails, the query plan will still execute up until the second fetch,
@@ -176,7 +179,7 @@ impl Variables {
                             .map(|(variable_key, value)| (variable_key.clone(), value.clone()))
                     })
                     .collect::(),
-                paths: HashMap::new(),
+                inverted_paths: Vec::new(),
             })
         }
     }
@@ -198,7 +201,10 @@ impl FetchNode {
             ..
         } = self;
 
-        let Variables { variables, paths } = match Variables::new(
+        let Variables {
+            variables,
+            inverted_paths: paths,
+        } = match Variables::new(
             &self.requires,
             self.variable_usages.as_ref(),
             data,
@@ -207,9 +213,7 @@ impl FetchNode {
             parameters.supergraph_request,
             parameters.schema,
             &self.input_rewrites,
-        )
-        .await
-        {
+        ) {
             Some(variables) => variables,
             None => {
                 return Ok((Value::Object(Object::default()), Vec::new()));
@@ -304,15 +308,9 @@ impl FetchNode {
         &'a self,
         schema: &Schema,
         current_dir: &'a Path,
-        paths: HashMap,
+        inverted_paths: Vec>,
         response: graphql::Response,
     ) -> (Value, Vec) {
-        // for each entity in the response, find out the path where it must be inserted
-        let mut inverted_paths: HashMap> = HashMap::new();
-        for (path, index) in paths.iter() {
-            (*inverted_paths.entry(*index).or_default()).push(path);
-        }
-
         if !self.requires.is_empty() {
             let entities_path = Path(vec![json_ext::PathElement::Key("_entities".to_string())]);
 
@@ -330,7 +328,7 @@ impl FetchNode {
                         match path.0.get(1) {
                             Some(json_ext::PathElement::Index(i)) => {
                                 for values_path in
-                                    inverted_paths.get(i).iter().flat_map(|v| v.iter())
+                                    inverted_paths.get(*i).iter().flat_map(|v| v.iter())
                                 {
                                     errors.push(Error {
                                         locations: error.locations.clone(),
@@ -370,7 +368,7 @@ impl FetchNode {
                         for (index, mut entity) in array.into_iter().enumerate() {
                             rewrites::apply_rewrites(schema, &mut entity, &self.output_rewrites);
 
-                            if let Some(paths) = inverted_paths.get(&index) {
+                            if let Some(paths) = inverted_paths.get(index) {
                                 if paths.len() > 1 {
                                     for path in &paths[1..] {
                                         let _ = value.insert(path, entity.clone());
diff --git a/apollo-router/src/query_planner/selection.rs b/apollo-router/src/query_planner/selection.rs
index 878ae72375..521882f40c 100644
--- a/apollo-router/src/query_planner/selection.rs
+++ b/apollo-router/src/query_planner/selection.rs
@@ -1,6 +1,6 @@
 use serde::Deserialize;
 use serde::Serialize;
-use serde_json_bytes::Entry;
+use serde_json_bytes::ByteString;
 
 use crate::error::FetchError;
 use crate::json_ext::Object;
@@ -53,16 +53,15 @@ pub(crate) fn select_object(
     selections: &[Selection],
     schema: &Schema,
 ) -> Result, FetchError> {
-    let mut output = Object::new();
+    let mut output = Object::with_capacity(selections.len());
     for selection in selections {
         match selection {
             Selection::Field(field) => {
-                if let Some(value) = select_field(content, field, schema)? {
-                    match output.entry(field.name.to_owned()) {
-                        Entry::Occupied(mut existing) => existing.get_mut().deep_merge(value),
-                        Entry::Vacant(vacant) => {
-                            vacant.insert(value);
-                        }
+                if let Some((key, value)) = select_field(content, field, schema)? {
+                    if let Some(o) = output.get_mut(field.name.as_str()) {
+                        o.deep_merge(value);
+                    } else {
+                        output.insert(key.to_owned(), value);
                     }
                 }
             }
@@ -81,13 +80,16 @@ pub(crate) fn select_object(
     Ok(Some(Value::Object(output)))
 }
 
-fn select_field(
-    content: &Object,
+fn select_field<'a>(
+    content: &'a Object,
     field: &Field,
     schema: &Schema,
-) -> Result, FetchError> {
-    let res = match (content.get(field.name.as_str()), &field.selections) {
-        (Some(v), _) => select_value(v, field, schema),
+) -> Result, FetchError> {
+    let res = match (
+        content.get_key_value(field.name.as_str()),
+        &field.selections,
+    ) {
+        (Some((k, v)), _) => select_value(v, field, schema).map(|opt| opt.map(|v| (k, v))),
         (None, _) => Err(FetchError::ExecutionFieldNotFound {
             field: field.name.to_owned(),
         }),
diff --git a/apollo-router/src/query_planner/subscription.rs b/apollo-router/src/query_planner/subscription.rs
index d91014b83e..d409c5b136 100644
--- a/apollo-router/src/query_planner/subscription.rs
+++ b/apollo-router/src/query_planner/subscription.rs
@@ -209,9 +209,7 @@ impl SubscriptionNode {
             parameters.supergraph_request,
             parameters.schema,
             &self.input_rewrites,
-        )
-        .await
-        {
+        ) {
             Some(variables) => variables,
             None => {
                 return Ok(Vec::new());

From 70cb943ed83555638e6cced2cd378e6f54299b2d Mon Sep 17 00:00:00 2001
From: David Glasser 
Date: Fri, 22 Sep 2023 08:17:55 -0700
Subject: [PATCH 39/51] persisted queries: improve Uplink failover and error
 handling (#3863)

If an Uplink request fails, the `stream_from_uplink` function
automatically tries the next configured Uplink endpoint. However, in the
case of the persisted queries feature, the Uplink response contains an
URL located in the same cloud provider as the Uplink endpoint (which is
used to fetch the full PQ list chunk). Before this PR, the PQ code
fetched this second URL "outside" of `stream_from_uplink`, so an error
downloading this file would *not* result in re-trying the Uplink request
itself on the next endpoint.

This PR factors out the body of `stream_from_uplink` into
`stream_from_uplink_transforming_new_response`, which takes an async
function that is called on any `UplinkResponse::New` to transform the
response. (`stream_from_uplink` passes the identity function here, so
its API is not affected.) If this function returns `Err`, the uplink
code moves on to the next endpoint URL, just like if the initial Uplink
fetch had failed. The PQ layer now fetches the PQ manifest chunk bodies
inside this async function instead of by mapping the uplink stream.

This means that if our GCP Uplink server is up but the GCS servers that
serve manifests from GCP are down, we will fail over to the AWS Uplink
server instead of just failing to fetch PQs.

Additionally, the response from Uplink can specify multiple valid URLs
for each chunk. (Uplink does not currently do this, but the protocol
allows for it.) Before this PR, Router would only look at the first URL
listed; after this PR, it tries each URL in order until finding one that
does not fail.

Additionally, before this PR, any errors fetching PQs from Uplink or GCS
after a successful startup were ignored. (They would be sent on the
`ready_sender` channel whose receiver is dropped, which could lead to a
`debug`-level logged message about how the channel wasn't open, but they
were not *directly* logged.) After this PR, we don't try to send these
errors on the channel after the channel was used once; instead, we log
them at the `error` level (similarly to how schema and license uplink
errors work). Note that this only occurs if Router fails to fetch PQs
from all configured Uplink endpoints; a failure to fetch from a single
endpoint is still only logged at the `debug` level.

Co-authored-by: Jeremy Lempereur 

---------

Co-authored-by: o0Ignition0o 
Co-authored-by: Jeremy Lempereur 
---
 ...ted_queries_failover_and_error_handling.md |   5 +
 .../persisted_queries/manifest_poller.rs      | 212 +++++++++++------
 .../test_harness/mocks/persisted_queries.rs   |  60 ++++-
 apollo-router/src/uplink/mod.rs               | 224 ++++++++++++++----
 ...sponse_first_response_transform_fails.snap |   6 +
 5 files changed, 371 insertions(+), 136 deletions(-)
 create mode 100644 .changesets/fix_persisted_queries_failover_and_error_handling.md
 create mode 100644 apollo-router/src/uplink/snapshots/apollo_router__uplink__test__stream_from_uplink_transforming_new_response_first_response_transform_fails.snap

diff --git a/.changesets/fix_persisted_queries_failover_and_error_handling.md b/.changesets/fix_persisted_queries_failover_and_error_handling.md
new file mode 100644
index 0000000000..554ec346f0
--- /dev/null
+++ b/.changesets/fix_persisted_queries_failover_and_error_handling.md
@@ -0,0 +1,5 @@
+### Improve multi-cloud failover and error handling for Persisted Queries
+
+Improves the resilience of the Persisted Queries feature to Uplink outages, and makes errors fetching Persisted Query Manifests from Uplink more visible.
+
+By [@glasser](https://github.com/glasser) in https://github.com/apollographql/router/pull/3863
\ No newline at end of file
diff --git a/apollo-router/src/services/layers/persisted_queries/manifest_poller.rs b/apollo-router/src/services/layers/persisted_queries/manifest_poller.rs
index 552536a18c..308cb5a440 100644
--- a/apollo-router/src/services/layers/persisted_queries/manifest_poller.rs
+++ b/apollo-router/src/services/layers/persisted_queries/manifest_poller.rs
@@ -19,7 +19,7 @@ use tower::BoxError;
 use crate::uplink::persisted_queries_manifest_stream::MaybePersistedQueriesManifestChunks;
 use crate::uplink::persisted_queries_manifest_stream::PersistedQueriesManifestChunk;
 use crate::uplink::persisted_queries_manifest_stream::PersistedQueriesManifestQuery;
-use crate::uplink::stream_from_uplink;
+use crate::uplink::stream_from_uplink_transforming_new_response;
 use crate::uplink::UplinkConfig;
 use crate::Configuration;
 
@@ -347,23 +347,32 @@ async fn poll_uplink(
     mut drop_receiver: mpsc::Receiver<()>,
     http_client: Client,
 ) {
+    let http_client = http_client.clone();
     let mut uplink_executor = stream::select_all(vec![
-        stream_from_uplink::(
-            uplink_config.clone(),
-        )
-        .filter_map(|res| {
+        stream_from_uplink_transforming_new_response::<
+            PersistedQueriesManifestQuery,
+            MaybePersistedQueriesManifestChunks,
+            Option,
+        >(uplink_config.clone(), move |response| {
             let http_client = http_client.clone();
-            let graph_ref = uplink_config.apollo_graph_ref.clone();
-            async move {
-                match res {
-                    Ok(Some(chunks)) => match manifest_from_chunks(chunks, http_client).await {
-                        Ok(new_manifest) => Some(ManifestPollEvent::NewManifest(new_manifest)),
-                        Err(e) => Some(ManifestPollEvent::FetchError(e)),
-                    },
-                    Ok(None) => Some(ManifestPollEvent::NoPersistedQueryList { graph_ref }),
-                    Err(e) => Some(ManifestPollEvent::Err(e.into())),
+            Box::new(Box::pin(async move {
+                match response {
+                    Some(chunks) => manifest_from_chunks(chunks, http_client)
+                        .await
+                        .map(Some)
+                        .map_err(|err| {
+                            format!("could not download persisted query lists: {}", err).into()
+                        }),
+                    None => Ok(None),
                 }
-            }
+            }))
+        })
+        .map(|res| match res {
+            Ok(Some(new_manifest)) => ManifestPollEvent::NewManifest(new_manifest),
+            Ok(None) => ManifestPollEvent::NoPersistedQueryList {
+                graph_ref: uplink_config.apollo_graph_ref.clone(),
+            },
+            Err(e) => ManifestPollEvent::Err(e.into()),
         })
         .boxed(),
         drop_receiver
@@ -373,7 +382,8 @@ async fn poll_uplink(
                 future::ready(match res {
                     None => Some(ManifestPollEvent::Shutdown),
                     Some(()) => Some(ManifestPollEvent::Err(
-                        "received message on drop channel in persisted query layer, which never gets sent"
+                        "received message on drop channel in persisted query layer, which never \
+                         gets sent"
                             .into(),
                     )),
                 })
@@ -383,7 +393,7 @@ async fn poll_uplink(
     .take_while(|msg| future::ready(!matches!(msg, ManifestPollEvent::Shutdown)))
     .boxed();
 
-    let mut resolved_first_pq_manifest = false;
+    let mut ready_sender_once = Some(ready_sender);
 
     while let Some(event) = uplink_executor.next().await {
         match event {
@@ -423,30 +433,22 @@ async fn poll_uplink(
                     })
                     .expect("could not acquire write lock on persisted query manifest state");
 
-                if !resolved_first_pq_manifest {
-                    send_startup_event(
-                        &ready_sender,
-                        ManifestPollResultOnStartup::LoadedOperations,
-                    )
-                    .await;
-                    resolved_first_pq_manifest = true;
-                }
-            }
-            ManifestPollEvent::FetchError(e) => {
-                send_startup_event(
-                    &ready_sender,
-                    ManifestPollResultOnStartup::Err(
-                        format!("could not fetch persisted queries: {e}").into(),
-                    ),
+                send_startup_event_or_log_error(
+                    &mut ready_sender_once,
+                    ManifestPollResultOnStartup::LoadedOperations,
                 )
-                .await
+                .await;
             }
             ManifestPollEvent::Err(e) => {
-                send_startup_event(&ready_sender, ManifestPollResultOnStartup::Err(e)).await
+                send_startup_event_or_log_error(
+                    &mut ready_sender_once,
+                    ManifestPollResultOnStartup::Err(e),
+                )
+                .await
             }
             ManifestPollEvent::NoPersistedQueryList { graph_ref } => {
-                send_startup_event(
-                    &ready_sender,
+                send_startup_event_or_log_error(
+                    &mut ready_sender_once,
                     ManifestPollResultOnStartup::Err(
                         format!("no persisted query list found for graph ref {}", &graph_ref)
                             .into(),
@@ -459,12 +461,28 @@ async fn poll_uplink(
         }
     }
 
-    async fn send_startup_event(
-        ready_sender: &mpsc::Sender,
+    async fn send_startup_event_or_log_error(
+        ready_sender: &mut Option>,
         message: ManifestPollResultOnStartup,
     ) {
-        if let Err(e) = ready_sender.send(message).await {
-            tracing::debug!("could not send startup event for the persisted query layer: {e}");
+        match (ready_sender.take(), message) {
+            (Some(ready_sender), message) => {
+                if let Err(e) = ready_sender.send(message).await {
+                    tracing::debug!(
+                        "could not send startup event for the persisted query layer: {e}"
+                    );
+                }
+            }
+            (None, ManifestPollResultOnStartup::Err(err)) => {
+                // We've already successfully started up, but we received some sort of error. This doesn't
+                // need to break our functional router, but we can log in case folks are interested.
+                tracing::error!(
+                    "error while polling uplink for persisted query manifests: {}",
+                    err
+                )
+            }
+            // Do nothing in the normal background "new manifest" case.
+            (None, ManifestPollResultOnStartup::LoadedOperations) => {}
         }
     }
 }
@@ -498,46 +516,69 @@ async fn add_chunk_to_operations(
     operations: &mut PersistedQueryManifest,
     http_client: Client,
 ) -> Result<(), BoxError> {
-    // TODO: chunk URLs will eventually respond with fallback URLs, when it does, implement falling back here
-    if let Some(chunk_url) = chunk.urls.get(0) {
-        let chunk = http_client
-            .get(chunk_url.clone())
-            .send()
-            .await
-            .and_then(|r| r.error_for_status())
-            .map_err(|e| -> BoxError {
-                format!(
-                    "error fetching persisted queries manifest chunk from {}: {}",
-                    chunk_url, e
-                )
-                .into()
-            })?
-            .json::()
-            .await
-            .map_err(|e| -> BoxError {
-                format!(
-                    "error reading body of persisted queries manifest chunk from {}: {}",
-                    chunk_url, e
-                )
-                .into()
-            })?;
-
-        if chunk.format != "apollo-persisted-query-manifest" {
-            return Err("chunk format is not 'apollo-persisted-query-manifest'".into());
-        }
-
-        if chunk.version != 1 {
-            return Err("persisted query manifest chunk version is not 1".into());
+    let mut it = chunk.urls.iter().peekable();
+    while let Some(chunk_url) = it.next() {
+        match fetch_chunk(http_client.clone(), chunk_url).await {
+            Ok(chunk) => {
+                for operation in chunk.operations {
+                    operations.insert(operation.id, operation.body);
+                }
+                return Ok(());
+            }
+            Err(e) => {
+                if it.peek().is_some() {
+                    // There's another URL to try, so log as debug and move on.
+                    tracing::debug!(
+                        "failed to fetch persisted query list chunk from {}: {}. \
+                         Other endpoints will be tried",
+                        chunk_url,
+                        e
+                    );
+                    continue;
+                } else {
+                    // No more URLs; fail the function.
+                    return Err(e);
+                }
+            }
         }
+    }
+    // The loop always returns unless there's another iteration after it, so the
+    // only way we can fall off the loop is if we never entered it.
+    Err("persisted query chunk did not include any URLs to fetch operations from".into())
+}
 
-        for operation in chunk.operations {
-            operations.insert(operation.id, operation.body);
-        }
+async fn fetch_chunk(http_client: Client, chunk_url: &String) -> Result {
+    let chunk = http_client
+        .get(chunk_url.clone())
+        .send()
+        .await
+        .and_then(|r| r.error_for_status())
+        .map_err(|e| -> BoxError {
+            format!(
+                "error fetching persisted queries manifest chunk from {}: {}",
+                chunk_url, e
+            )
+            .into()
+        })?
+        .json::()
+        .await
+        .map_err(|e| -> BoxError {
+            format!(
+                "error reading body of persisted queries manifest chunk from {}: {}",
+                chunk_url, e
+            )
+            .into()
+        })?;
+
+    if chunk.format != "apollo-persisted-query-manifest" {
+        return Err("chunk format is not 'apollo-persisted-query-manifest'".into());
+    }
 
-        Ok(())
-    } else {
-        Err("persisted query chunk did not include any URLs to fetch operations from".into())
+    if chunk.version != 1 {
+        return Err("persisted query manifest chunk version is not 1".into());
     }
+
+    Ok(chunk)
 }
 
 /// Types of events produced by the manifest poller.
@@ -546,7 +587,6 @@ pub(crate) enum ManifestPollEvent {
     NewManifest(PersistedQueryManifest),
     NoPersistedQueryList { graph_ref: String },
     Err(BoxError),
-    FetchError(BoxError),
     Shutdown,
 }
 
@@ -610,6 +650,24 @@ mod tests {
         .is_err());
     }
 
+    #[tokio::test(flavor = "multi_thread")]
+    async fn poller_fails_over_on_gcs_failure() {
+        let (_mock_server1, url1) = mock_pq_uplink_bad_gcs().await;
+        let (id, body, manifest) = fake_manifest();
+        let (_mock_guard2, url2) = mock_pq_uplink_one_endpoint(&manifest, None).await;
+        let manifest_manager = PersistedQueryManifestPoller::new(
+            Configuration::fake_builder()
+                .uplink(UplinkConfig::for_tests(Endpoints::fallback(vec![
+                    url1, url2,
+                ])))
+                .build()
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+        assert_eq!(manifest_manager.get_operation_body(&id), Some(body))
+    }
+
     #[test]
     fn safelist_body_normalization() {
         let safelist = FreeformGraphQLSafelist::new(&PersistedQueryManifest::from([(
diff --git a/apollo-router/src/test_harness/mocks/persisted_queries.rs b/apollo-router/src/test_harness/mocks/persisted_queries.rs
index 822c91bf59..1e0cede210 100644
--- a/apollo-router/src/test_harness/mocks/persisted_queries.rs
+++ b/apollo-router/src/test_harness/mocks/persisted_queries.rs
@@ -32,12 +32,20 @@ pub async fn mock_pq_uplink_with_delay(
     manifest: &HashMap,
     delay: Duration,
 ) -> (UplinkMockGuard, UplinkConfig) {
-    do_mock_pq_uplink(manifest, Some(delay)).await
+    let (guard, url) = mock_pq_uplink_one_endpoint(manifest, Some(delay)).await;
+    (
+        guard,
+        UplinkConfig::for_tests(Endpoints::fallback(vec![url])),
+    )
 }
 
 /// Mocks an uplink server with a persisted query list containing operations passed to this function.
 pub async fn mock_pq_uplink(manifest: &HashMap) -> (UplinkMockGuard, UplinkConfig) {
-    do_mock_pq_uplink(manifest, None).await
+    let (guard, url) = mock_pq_uplink_one_endpoint(manifest, None).await;
+    (
+        guard,
+        UplinkConfig::for_tests(Endpoints::fallback(vec![url])),
+    )
 }
 
 /// Guards for the uplink and GCS mock servers, dropping these structs shuts down the server.
@@ -52,10 +60,12 @@ struct Operation {
     body: String,
 }
 
-async fn do_mock_pq_uplink(
+/// Mocks an uplink server; returns a single Url rather than a full UplinkConfig, so you
+/// can combine it with another one to test failover.
+pub async fn mock_pq_uplink_one_endpoint(
     manifest: &HashMap,
     delay: Option,
-) -> (UplinkMockGuard, UplinkConfig) {
+) -> (UplinkMockGuard, Url) {
     let operations: Vec = manifest
         // clone the manifest so the caller can still make assertions about it
         .clone()
@@ -80,7 +90,7 @@ async fn do_mock_pq_uplink(
 
     let mock_uplink_server = MockServer::start().await;
 
-    let mut gcs_response = ResponseTemplate::new(200).set_body_json(json!({
+    let mut uplink_response = ResponseTemplate::new(200).set_body_json(json!({
           "data": {
             "persistedQueries": {
               "__typename": "PersistedQueriesResult",
@@ -90,7 +100,8 @@ async fn do_mock_pq_uplink(
                 {
                   "id": "graph-id/889406a1-b4f8-44df-a499-6c1e3c1bea09/ec8ae3ae3eb00c738031dbe81603489b5d24fbf58f15bdeec1587282ee4e6eea",
                   "urls": [
-                    mock_gcs_server_uri
+                    "https://a.broken.gcs.url.that.will.get.fetched.and.skipped.unknown/",
+                    mock_gcs_server_uri,
                   ]
                 }
               ]
@@ -99,11 +110,11 @@ async fn do_mock_pq_uplink(
         }));
 
     if let Some(delay) = delay {
-        gcs_response = gcs_response.set_delay(delay);
+        uplink_response = uplink_response.set_delay(delay);
     }
 
     Mock::given(method("POST"))
-        .respond_with(gcs_response)
+        .respond_with(uplink_response)
         .mount(&mock_uplink_server)
         .await;
 
@@ -113,6 +124,37 @@ async fn do_mock_pq_uplink(
             _uplink_mock_guard: mock_uplink_server,
             _gcs_mock_guard: mock_gcs_server,
         },
-        UplinkConfig::for_tests(Endpoints::fallback(vec![url])),
+        url,
     )
 }
+
+/// Mocks an uplink server which returns bad GCS URLs.
+pub async fn mock_pq_uplink_bad_gcs() -> (MockServer, Url) {
+    let mock_uplink_server = MockServer::start().await;
+
+    let  uplink_response = ResponseTemplate::new(200).set_body_json(json!({
+          "data": {
+            "persistedQueries": {
+              "__typename": "PersistedQueriesResult",
+              "id": "889406d7-b4f8-44df-a499-6c1e3c1bea09:1",
+              "minDelaySeconds": 60,
+              "chunks": [
+                {
+                  "id": "graph-id/889406a1-b4f8-44df-a499-6c1e3c1bea09/ec8ae3ae3eb00c738031dbe81603489b5d24fbf58f15bdeec1587282ee4e6eea",
+                  "urls": [
+                    "https://definitely.not.gcs.unknown"
+                  ]
+                }
+              ]
+            }
+          }
+        }));
+
+    Mock::given(method("POST"))
+        .respond_with(uplink_response)
+        .mount(&mock_uplink_server)
+        .await;
+
+    let url = mock_uplink_server.uri().parse().unwrap();
+    (mock_uplink_server, url)
+}
diff --git a/apollo-router/src/uplink/mod.rs b/apollo-router/src/uplink/mod.rs
index 1399a6b606..bdb329d438 100644
--- a/apollo-router/src/uplink/mod.rs
+++ b/apollo-router/src/uplink/mod.rs
@@ -3,12 +3,14 @@ use std::fmt::Debug;
 use std::time::Duration;
 use std::time::Instant;
 
+use futures::Future;
 use futures::Stream;
 use futures::StreamExt;
 use graphql_client::QueryBody;
 use thiserror::Error;
 use tokio::sync::mpsc::channel;
 use tokio_stream::wrappers::ReceiverStream;
+use tower::BoxError;
 use tracing::instrument::WithSubscriber;
 use url::Url;
 
@@ -160,13 +162,43 @@ impl UplinkConfig {
 /// Regularly fetch from Uplink
 /// If urls are supplied then they will be called round robin
 pub(crate) fn stream_from_uplink(
-    mut uplink_config: UplinkConfig,
+    uplink_config: UplinkConfig,
 ) -> impl Stream>
 where
     Query: graphql_client::GraphQLQuery,
     ::ResponseData: Into> + Send,
     ::Variables: From + Send + Sync,
     Response: Send + 'static + Debug,
+{
+    stream_from_uplink_transforming_new_response::(
+        uplink_config,
+        |response| Box::new(Box::pin(async { Ok(response) })),
+    )
+}
+
+/// Like stream_from_uplink, but applies an async transformation function to the
+/// result of the HTTP fetch if the response is an UplinkResponse::New. If this
+/// function returns Err, we fail over to the next Uplink endpoint, just like if
+/// the HTTP fetch itself failed. This serves the use case where an Uplink
+/// endpoint's response includes another URL located close to the Uplink
+/// endpoint; if that second URL is down, we want to try the next Uplink
+/// endpoint rather than fully giving up.
+pub(crate) fn stream_from_uplink_transforming_new_response(
+    mut uplink_config: UplinkConfig,
+    transform_new_response: impl Fn(
+            Response,
+        )
+            -> Box> + Send + Unpin>
+        + Send
+        + Sync
+        + 'static,
+) -> impl Stream>
+where
+    Query: graphql_client::GraphQLQuery,
+    ::ResponseData: Into> + Send,
+    ::Variables: From + Send + Sync,
+    Response: Send + 'static + Debug,
+    TransformedResponse: Send + 'static + Debug,
 {
     let query = query_name::();
     let (sender, receiver) = channel(2);
@@ -193,7 +225,14 @@ where
 
             let query_body = Query::build_query(variables.into());
 
-            match fetch::(&client, &query_body, &mut endpoints.iter()).await {
+            match fetch::(
+                &client,
+                &query_body,
+                &mut endpoints.iter(),
+                &transform_new_response,
+            )
+            .await
+            {
                 Ok(response) => {
                     tracing::info!(
                         monotonic_counter.apollo_router_uplink_fetch_count_total = 1u64,
@@ -264,68 +303,102 @@ where
     ReceiverStream::new(receiver).boxed()
 }
 
-pub(crate) async fn fetch(
+pub(crate) async fn fetch(
     client: &reqwest::Client,
     request_body: &QueryBody,
     urls: &mut impl Iterator,
-) -> Result, Error>
+    // See stream_from_uplink_transforming_new_response for an explanation of
+    // this argument.
+    transform_new_response: &(impl Fn(
+        Response,
+    ) -> Box> + Send + Unpin>
+          + Send
+          + Sync
+          + 'static),
+) -> Result, Error>
 where
     Query: graphql_client::GraphQLQuery,
     ::ResponseData: Into> + Send,
     ::Variables: From + Send + Sync,
     Response: Send + Debug + 'static,
+    TransformedResponse: Send + Debug + 'static,
 {
     let query = query_name::();
     for url in urls {
         let now = Instant::now();
         match http_request::(client, url.as_str(), request_body).await {
-            Ok(response) => {
-                let response = response.data.map(Into::into);
-                match &response {
-                    None => {
-                        tracing::info!(
-                            histogram.apollo_router_uplink_fetch_duration_seconds =
-                                now.elapsed().as_secs_f64(),
-                            query,
-                            url = url.to_string(),
-                            "kind" = "uplink_error",
-                            error = "empty response from uplink",
-                        );
-                    }
-                    Some(UplinkResponse::New { .. }) => {
-                        tracing::info!(
-                            histogram.apollo_router_uplink_fetch_duration_seconds =
-                                now.elapsed().as_secs_f64(),
-                            query,
-                            url = url.to_string(),
-                            "kind" = "new"
-                        );
-                        return Ok(response.expect("we are in the some branch, qed"));
-                    }
-                    Some(UplinkResponse::Unchanged { .. }) => {
-                        tracing::info!(
-                            histogram.apollo_router_uplink_fetch_duration_seconds =
-                                now.elapsed().as_secs_f64(),
-                            query,
-                            url = url.to_string(),
-                            "kind" = "unchanged"
-                        );
-                        return Ok(response.expect("we are in the some branch, qed"));
-                    }
-                    Some(UplinkResponse::Error { message, code, .. }) => {
-                        tracing::info!(
-                            histogram.apollo_router_uplink_fetch_duration_seconds =
-                                now.elapsed().as_secs_f64(),
-                            query,
-                            url = url.to_string(),
-                            "kind" = "uplink_error",
-                            error = message,
-                            code
-                        );
-                        return Ok(response.expect("we are in the some branch, qed"));
+            Ok(response) => match response.data.map(Into::into) {
+                None => {
+                    tracing::info!(
+                        histogram.apollo_router_uplink_fetch_duration_seconds =
+                            now.elapsed().as_secs_f64(),
+                        query,
+                        url = url.to_string(),
+                        "kind" = "uplink_error",
+                        error = "empty response from uplink",
+                    );
+                }
+                Some(UplinkResponse::New {
+                    response,
+                    id,
+                    delay,
+                }) => {
+                    tracing::info!(
+                        histogram.apollo_router_uplink_fetch_duration_seconds =
+                            now.elapsed().as_secs_f64(),
+                        query,
+                        url = url.to_string(),
+                        "kind" = "new"
+                    );
+                    match transform_new_response(response).await {
+                        Ok(res) => {
+                            return Ok(UplinkResponse::New {
+                                response: res,
+                                id,
+                                delay,
+                            })
+                        }
+                        Err(err) => {
+                            tracing::debug!(
+                                    "failed to process results of Uplink response from {}: {}. Other endpoints will be tried",
+                                    url,
+                                    err
+                                );
+                            continue;
+                        }
                     }
                 }
-            }
+                Some(UplinkResponse::Unchanged { id, delay }) => {
+                    tracing::info!(
+                        histogram.apollo_router_uplink_fetch_duration_seconds =
+                            now.elapsed().as_secs_f64(),
+                        query,
+                        url = url.to_string(),
+                        "kind" = "unchanged"
+                    );
+                    return Ok(UplinkResponse::Unchanged { id, delay });
+                }
+                Some(UplinkResponse::Error {
+                    message,
+                    code,
+                    retry_later,
+                }) => {
+                    tracing::info!(
+                        histogram.apollo_router_uplink_fetch_duration_seconds =
+                            now.elapsed().as_secs_f64(),
+                        query,
+                        url = url.to_string(),
+                        "kind" = "uplink_error",
+                        error = message,
+                        code
+                    );
+                    return Ok(UplinkResponse::Error {
+                        message,
+                        code,
+                        retry_later,
+                    });
+                }
+            },
             Err(e) => {
                 tracing::info!(
                     histogram.apollo_router_uplink_fetch_duration_seconds =
@@ -415,6 +488,7 @@ mod test {
     use wiremock::ResponseTemplate;
 
     use crate::uplink::stream_from_uplink;
+    use crate::uplink::stream_from_uplink_transforming_new_response;
     use crate::uplink::Endpoints;
     use crate::uplink::Error;
     use crate::uplink::UplinkConfig;
@@ -437,6 +511,13 @@ mod test {
         ordering: i64,
     }
 
+    #[allow(dead_code)]
+    #[derive(Debug)]
+    struct TransformedQueryResult {
+        name: String,
+        halved_ordering: i64,
+    }
+
     impl From for test_query::Variables {
         fn from(req: UplinkRequest) -> Self {
             test_query::Variables {
@@ -789,7 +870,50 @@ mod test {
         assert_yaml_snapshot!(results.into_iter().map(to_friendly).collect::>());
     }
 
-    fn to_friendly(r: Result) -> Result {
+    #[tokio::test(flavor = "multi_thread")]
+    async fn stream_from_uplink_transforming_new_response_first_response_transform_fails() {
+        let (mock_server, url1, url2, _url3) = init_mock_server().await;
+        MockResponses::builder()
+            .mock_server(&mock_server)
+            .endpoint(&url1)
+            .response(response_ok(15))
+            .build()
+            .await;
+        MockResponses::builder()
+            .mock_server(&mock_server)
+            .endpoint(&url2)
+            .response(response_ok(100))
+            .build()
+            .await;
+        let results = stream_from_uplink_transforming_new_response::<
+            TestQuery,
+            QueryResult,
+            TransformedQueryResult,
+        >(
+            mock_uplink_config_with_fallback_urls(vec![url1, url2]),
+            |result| {
+                Box::new(Box::pin(async move {
+                    let QueryResult { name, ordering } = result;
+                    if ordering % 2 == 0 {
+                        // This will trigger on url2's response.
+                        Ok(TransformedQueryResult {
+                            name,
+                            halved_ordering: ordering / 2,
+                        })
+                    } else {
+                        // This will trigger on url1's response.
+                        Err("cannot halve an odd number".into())
+                    }
+                }))
+            },
+        )
+        .take(1)
+        .collect::>()
+        .await;
+        assert_yaml_snapshot!(results.into_iter().map(to_friendly).collect::>());
+    }
+
+    fn to_friendly(r: Result) -> Result {
         match r {
             Ok(e) => Ok(format!("result {:?}", e)),
             Err(e) => Err(e.to_string()),
diff --git a/apollo-router/src/uplink/snapshots/apollo_router__uplink__test__stream_from_uplink_transforming_new_response_first_response_transform_fails.snap b/apollo-router/src/uplink/snapshots/apollo_router__uplink__test__stream_from_uplink_transforming_new_response_first_response_transform_fails.snap
new file mode 100644
index 0000000000..c798bdb37b
--- /dev/null
+++ b/apollo-router/src/uplink/snapshots/apollo_router__uplink__test__stream_from_uplink_transforming_new_response_first_response_transform_fails.snap
@@ -0,0 +1,6 @@
+---
+source: apollo-router/src/uplink/mod.rs
+expression: "results.into_iter().map(to_friendly).collect::>()"
+---
+- Ok: "result TransformedQueryResult { name: \"ok\", halved_ordering: 50 }"
+

From 03414256123c57707b139c9a47324e79f26f564a Mon Sep 17 00:00:00 2001
From: Jesse Rosenberger 
Date: Fri, 22 Sep 2023 15:40:41 +0000
Subject: [PATCH 40/51] prep release: v1.31.0-alpha.1

---
 Cargo.lock                                    |  6 ++--
 apollo-router-benchmarks/Cargo.toml           |  2 +-
 apollo-router-scaffold/Cargo.toml             |  2 +-
 .../templates/base/Cargo.toml                 |  2 +-
 .../templates/base/xtask/Cargo.toml           |  2 +-
 apollo-router/Cargo.toml                      |  2 +-
 .../tracing/docker-compose.datadog.yml        |  2 +-
 dockerfiles/tracing/docker-compose.jaeger.yml |  2 +-
 dockerfiles/tracing/docker-compose.zipkin.yml |  2 +-
 docs/source/containerization/docker.mdx       |  2 +-
 docs/source/containerization/kubernetes.mdx   | 28 +++++++++----------
 helm/chart/router/Chart.yaml                  |  4 +--
 helm/chart/router/README.md                   |  6 ++--
 licenses.html                                 |  4 +--
 scripts/install.sh                            |  2 +-
 15 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index acb00b7a0a..a8ebb7ba9b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -264,7 +264,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router"
-version = "1.31.0-alpha.0"
+version = "1.31.0-alpha.1"
 dependencies = [
  "access-json",
  "anyhow",
@@ -411,7 +411,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.0"
+version = "1.31.0-alpha.1"
 dependencies = [
  "apollo-parser 0.6.2",
  "apollo-router",
@@ -427,7 +427,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.0"
+version = "1.31.0-alpha.1"
 dependencies = [
  "anyhow",
  "cargo-scaffold",
diff --git a/apollo-router-benchmarks/Cargo.toml b/apollo-router-benchmarks/Cargo.toml
index 4620aa59f5..5a0e3de182 100644
--- a/apollo-router-benchmarks/Cargo.toml
+++ b/apollo-router-benchmarks/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.0"
+version = "1.31.0-alpha.1"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml
index d6ba4afe51..f3feed2de0 100644
--- a/apollo-router-scaffold/Cargo.toml
+++ b/apollo-router-scaffold/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.0"
+version = "1.31.0-alpha.1"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/templates/base/Cargo.toml b/apollo-router-scaffold/templates/base/Cargo.toml
index 0f8247ae22..8bd810d4a9 100644
--- a/apollo-router-scaffold/templates/base/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/Cargo.toml
@@ -22,7 +22,7 @@ apollo-router = { path ="{{integration_test}}apollo-router" }
 apollo-router = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
 # Note if you update these dependencies then also update xtask/Cargo.toml
-apollo-router = "1.31.0-alpha.0"
+apollo-router = "1.31.0-alpha.1"
 {{/if}}
 {{/if}}
 async-trait = "0.1.52"
diff --git a/apollo-router-scaffold/templates/base/xtask/Cargo.toml b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
index 904ff767ca..196b842527 100644
--- a/apollo-router-scaffold/templates/base/xtask/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
@@ -13,7 +13,7 @@ apollo-router-scaffold = { path ="{{integration_test}}apollo-router-scaffold" }
 {{#if branch}}
 apollo-router-scaffold = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
-apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.0" }
+apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.1" }
 {{/if}}
 {{/if}}
 anyhow = "1.0.58"
diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml
index 65fbd4e7d1..083bbef1c6 100644
--- a/apollo-router/Cargo.toml
+++ b/apollo-router/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router"
-version = "1.31.0-alpha.0"
+version = "1.31.0-alpha.1"
 authors = ["Apollo Graph, Inc. "]
 repository = "https://github.com/apollographql/router/"
 documentation = "https://docs.rs/apollo-router"
diff --git a/dockerfiles/tracing/docker-compose.datadog.yml b/dockerfiles/tracing/docker-compose.datadog.yml
index 4d3f5ee0bc..a3f7c92ada 100644
--- a/dockerfiles/tracing/docker-compose.datadog.yml
+++ b/dockerfiles/tracing/docker-compose.datadog.yml
@@ -3,7 +3,7 @@ services:
 
   apollo-router:
     container_name: apollo-router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.0
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.1
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/datadog.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.jaeger.yml b/dockerfiles/tracing/docker-compose.jaeger.yml
index bb2226979f..e3ae5c9f4f 100644
--- a/dockerfiles/tracing/docker-compose.jaeger.yml
+++ b/dockerfiles/tracing/docker-compose.jaeger.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     #build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.0
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.1
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/jaeger.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.zipkin.yml b/dockerfiles/tracing/docker-compose.zipkin.yml
index 191b008688..6db36f066c 100644
--- a/dockerfiles/tracing/docker-compose.zipkin.yml
+++ b/dockerfiles/tracing/docker-compose.zipkin.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.0
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.1
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/zipkin.router.yaml:/etc/config/configuration.yaml
diff --git a/docs/source/containerization/docker.mdx b/docs/source/containerization/docker.mdx
index 78e852ea46..5440d1c56f 100644
--- a/docs/source/containerization/docker.mdx
+++ b/docs/source/containerization/docker.mdx
@@ -11,7 +11,7 @@ The default behaviour of the router images is suitable for a quickstart or devel
 
 Note: The [docker documentation](https://docs.docker.com/engine/reference/run/) for the run command may be helpful when reading through the examples.
 
-Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.0`
+Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.1`
 
 ## Override the configuration
 
diff --git a/docs/source/containerization/kubernetes.mdx b/docs/source/containerization/kubernetes.mdx
index 0b82e0d26a..bda87c8353 100644
--- a/docs/source/containerization/kubernetes.mdx
+++ b/docs/source/containerization/kubernetes.mdx
@@ -13,7 +13,7 @@ import { Link } from 'gatsby';
 
 [Helm](https://helm.sh) is the package manager for kubernetes.
 
-There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.0/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
+There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.1/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
 
 In both the following examples, we are using helm to install the router:
  - into namespace "router-deploy" (create namespace if it doesn't exist)
@@ -64,10 +64,10 @@ kind: ServiceAccount
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.0
+    helm.sh/chart: router-1.31.0-alpha.1
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.0"
+    app.kubernetes.io/version: "v1.31.0-alpha.1"
     app.kubernetes.io/managed-by: Helm
 ---
 # Source: router/templates/secret.yaml
@@ -76,10 +76,10 @@ kind: Secret
 metadata:
   name: "release-name-router"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.0
+    helm.sh/chart: router-1.31.0-alpha.1
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.0"
+    app.kubernetes.io/version: "v1.31.0-alpha.1"
     app.kubernetes.io/managed-by: Helm
 data:
   managedFederationApiKey: "UkVEQUNURUQ="
@@ -90,10 +90,10 @@ kind: ConfigMap
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.0
+    helm.sh/chart: router-1.31.0-alpha.1
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.0"
+    app.kubernetes.io/version: "v1.31.0-alpha.1"
     app.kubernetes.io/managed-by: Helm
 data:
   configuration.yaml: |
@@ -117,10 +117,10 @@ kind: Service
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.0
+    helm.sh/chart: router-1.31.0-alpha.1
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.0"
+    app.kubernetes.io/version: "v1.31.0-alpha.1"
     app.kubernetes.io/managed-by: Helm
 spec:
   type: ClusterIP
@@ -143,10 +143,10 @@ kind: Deployment
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.0
+    helm.sh/chart: router-1.31.0-alpha.1
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.0"
+    app.kubernetes.io/version: "v1.31.0-alpha.1"
     app.kubernetes.io/managed-by: Helm
   
   annotations:
@@ -174,7 +174,7 @@ spec:
         - name: router
           securityContext:
             {}
-          image: "ghcr.io/apollographql/router:v1.31.0-alpha.0"
+          image: "ghcr.io/apollographql/router:v1.31.0-alpha.1"
           imagePullPolicy: IfNotPresent
           args:
             - --hot-reload
@@ -226,10 +226,10 @@ kind: Pod
 metadata:
   name: "release-name-router-test-connection"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.0
+    helm.sh/chart: router-1.31.0-alpha.1
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.0"
+    app.kubernetes.io/version: "v1.31.0-alpha.1"
     app.kubernetes.io/managed-by: Helm
   annotations:
     "helm.sh/hook": test
diff --git a/helm/chart/router/Chart.yaml b/helm/chart/router/Chart.yaml
index c43ce198b6..a66745badc 100644
--- a/helm/chart/router/Chart.yaml
+++ b/helm/chart/router/Chart.yaml
@@ -20,10 +20,10 @@ type: application
 # so it matches the shape of our release process and release automation.
 # By proxy of that decision, this version uses SemVer 2.0.0, though the prefix
 # of "v" is not included.
-version: 1.31.0-alpha.0
+version: 1.31.0-alpha.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.31.0-alpha.0"
+appVersion: "v1.31.0-alpha.1"
diff --git a/helm/chart/router/README.md b/helm/chart/router/README.md
index eae73cda3b..56de45829f 100644
--- a/helm/chart/router/README.md
+++ b/helm/chart/router/README.md
@@ -2,7 +2,7 @@
 
 [router](https://github.com/apollographql/router) Rust Graph Routing runtime for Apollo Federation
 
-![Version: 1.31.0-alpha.0](https://img.shields.io/badge/Version-1.31.0--alpha.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.0](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.0-informational?style=flat-square)
+![Version: 1.31.0-alpha.1](https://img.shields.io/badge/Version-1.31.0--alpha.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.1](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.1-informational?style=flat-square)
 
 ## Prerequisites
 
@@ -11,7 +11,7 @@
 ## Get Repo Info
 
 ```console
-helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.0
+helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.1
 ```
 
 ## Install Chart
@@ -19,7 +19,7 @@ helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.
 **Important:** only helm3 is supported
 
 ```console
-helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.0 --values my-values.yaml
+helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.1 --values my-values.yaml
 ```
 
 _See [configuration](#configuration) below._
diff --git a/licenses.html b/licenses.html
index 9f62423b2f..a77c5f6ba3 100644
--- a/licenses.html
+++ b/licenses.html
@@ -10493,11 +10493,9 @@ 
    Used by:
    
                 
    Apache License 2.0
    
                 
    Used by:
    
                 
                 
    ../../LICENSE-APACHE
    
@@ -11149,6 +11147,8 @@ 
    Used by:
    
                 
    Apache License 2.0
    
                 
    Used by:
    
                 
      
+                    
    • apollo-compiler
      • 
+                    
    • apollo-parser
      • 
                     
    • curve25519-dalek-derive
      • 
                     
    • deadpool-runtime
      • 
                     
    • deno-proc-macro-rules
      • 
diff --git a/scripts/install.sh b/scripts/install.sh
index 4d9c0ead7e..20aa4309f4 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -11,7 +11,7 @@ BINARY_DOWNLOAD_PREFIX="https://github.com/apollographql/router/releases/downloa
 
 # Router version defined in apollo-router's Cargo.toml
 # Note: Change this line manually during the release steps.
-PACKAGE_VERSION="v1.31.0-alpha.0"
+PACKAGE_VERSION="v1.31.0-alpha.1"
 
 download_binary() {
     downloader --check

From 8464e85d25119cd496c6a02ef3783e84d6914805 Mon Sep 17 00:00:00 2001
From: Jesse Rosenberger 
Date: Mon, 25 Sep 2023 05:21:40 +0000
Subject: [PATCH 41/51] Lint Fix

---
 apollo-router/src/router_factory.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apollo-router/src/router_factory.rs b/apollo-router/src/router_factory.rs
index dcab1ba737..55b44a8edd 100644
--- a/apollo-router/src/router_factory.rs
+++ b/apollo-router/src/router_factory.rs
@@ -34,9 +34,9 @@ use crate::plugins::traffic_shaping::RetryPolicy;
 use crate::plugins::traffic_shaping::TrafficShaping;
 use crate::plugins::traffic_shaping::APOLLO_TRAFFIC_SHAPING;
 use crate::query_planner::BridgeQueryPlanner;
-use crate::services::layers::persisted_queries::PersistedQueryLayer;
 use crate::services::apollo_graph_reference;
 use crate::services::apollo_key;
+use crate::services::layers::persisted_queries::PersistedQueryLayer;
 use crate::services::layers::query_analysis::QueryAnalysisLayer;
 use crate::services::new_service::ServiceFactory;
 use crate::services::router;

From 948706eee5dc76fa1f9267c083a3b6b001df850e Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Mon, 25 Sep 2023 06:33:46 +0000
Subject: [PATCH 42/51] chore(deps): update rust crate rhai to 1.16.2

---
 Cargo.lock               | 4 ++--
 apollo-router/Cargo.toml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5bd770be49..d59c5073d2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5063,9 +5063,9 @@ dependencies = [
 
 [[package]]
 name = "rhai"
-version = "1.16.1"
+version = "1.16.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "637a4f79f65571b1fd1a0ebbae05bbbf58a01faf612abbc3eea15cda34f0b87a"
+checksum = "206cee941730eaf90a22c84235b25193df661393688162e15551164f92f09eca"
 dependencies = [
  "ahash",
  "bitflags 2.4.0",
diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml
index bc77a18b22..70253a7f4f 100644
--- a/apollo-router/Cargo.toml
+++ b/apollo-router/Cargo.toml
@@ -165,7 +165,7 @@ prost-types = "0.11.9"
 proteus = "0.5.0"
 rand = "0.8.5"
 rand_core = "0.6.4"
-rhai = { version = "1.16.1", features = ["sync", "serde", "internals"] }
+rhai = { version = "1.16.2", features = ["sync", "serde", "internals"] }
 regex = "1.9.5"
 reqwest = { version = "0.11.19", default-features = false, features = [
     "rustls-tls",
@@ -274,7 +274,7 @@ reqwest = { version = "0.11.19", default-features = false, features = [
     "json",
     "stream",
 ] }
-rhai = { version = "1.16.1", features = [
+rhai = { version = "1.16.2", features = [
     "sync",
     "serde",
     "internals",

From 20b727b2ea8bec350802b5b87aebdf29df572308 Mon Sep 17 00:00:00 2001
From: Jesse Rosenberger 
Date: Mon, 25 Sep 2023 11:44:52 +0300
Subject: [PATCH 43/51] chore(ci): Gate release on pre_verify_release step
 (#3888)

Follows-up: https://github.com/apollographql/router/pull/3859

This builds on the successful deployment of the `pre_verify_release`
step but makes it an actual pre-flight checklist rather than something
that runs in parallel to the existing jobs.
---
 .circleci/config.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index c7db107850..3a91555b3f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -868,6 +868,7 @@ workflows:
             tags:
               only: /v.*/
       - build_release:
+          requires: [ pre_verify_release ]
           matrix:
             parameters:
               platform:

From 0ff8c4539213e458a5dc7f69f91cd9bd03a8f421 Mon Sep 17 00:00:00 2001
From: Jesse Rosenberger 
Date: Mon, 25 Sep 2023 12:37:03 +0000
Subject: [PATCH 44/51] prep release: v1.31.0-alpha.2

---
 Cargo.lock                                    |  6 ++--
 apollo-router-benchmarks/Cargo.toml           |  2 +-
 apollo-router-scaffold/Cargo.toml             |  2 +-
 .../templates/base/Cargo.toml                 |  2 +-
 .../templates/base/xtask/Cargo.toml           |  2 +-
 apollo-router/Cargo.toml                      |  2 +-
 .../tracing/docker-compose.datadog.yml        |  2 +-
 dockerfiles/tracing/docker-compose.jaeger.yml |  2 +-
 dockerfiles/tracing/docker-compose.zipkin.yml |  2 +-
 docs/source/containerization/docker.mdx       |  2 +-
 docs/source/containerization/kubernetes.mdx   | 28 +++++++++----------
 helm/chart/router/Chart.yaml                  |  4 +--
 helm/chart/router/README.md                   |  6 ++--
 scripts/install.sh                            |  2 +-
 14 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ee8a6b273c..a681a7bbb6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -264,7 +264,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router"
-version = "1.31.0-alpha.1"
+version = "1.31.0-alpha.2"
 dependencies = [
  "access-json",
  "anyhow",
@@ -411,7 +411,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.1"
+version = "1.31.0-alpha.2"
 dependencies = [
  "apollo-parser 0.6.2",
  "apollo-router",
@@ -427,7 +427,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.1"
+version = "1.31.0-alpha.2"
 dependencies = [
  "anyhow",
  "cargo-scaffold",
diff --git a/apollo-router-benchmarks/Cargo.toml b/apollo-router-benchmarks/Cargo.toml
index 5a0e3de182..1b12fa92cf 100644
--- a/apollo-router-benchmarks/Cargo.toml
+++ b/apollo-router-benchmarks/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.1"
+version = "1.31.0-alpha.2"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml
index f3feed2de0..47ac91e956 100644
--- a/apollo-router-scaffold/Cargo.toml
+++ b/apollo-router-scaffold/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.1"
+version = "1.31.0-alpha.2"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/templates/base/Cargo.toml b/apollo-router-scaffold/templates/base/Cargo.toml
index 8bd810d4a9..aab266039c 100644
--- a/apollo-router-scaffold/templates/base/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/Cargo.toml
@@ -22,7 +22,7 @@ apollo-router = { path ="{{integration_test}}apollo-router" }
 apollo-router = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
 # Note if you update these dependencies then also update xtask/Cargo.toml
-apollo-router = "1.31.0-alpha.1"
+apollo-router = "1.31.0-alpha.2"
 {{/if}}
 {{/if}}
 async-trait = "0.1.52"
diff --git a/apollo-router-scaffold/templates/base/xtask/Cargo.toml b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
index 196b842527..9c6341051d 100644
--- a/apollo-router-scaffold/templates/base/xtask/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
@@ -13,7 +13,7 @@ apollo-router-scaffold = { path ="{{integration_test}}apollo-router-scaffold" }
 {{#if branch}}
 apollo-router-scaffold = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
-apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.1" }
+apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.2" }
 {{/if}}
 {{/if}}
 anyhow = "1.0.58"
diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml
index 54cb8a6c48..1aeb8de459 100644
--- a/apollo-router/Cargo.toml
+++ b/apollo-router/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router"
-version = "1.31.0-alpha.1"
+version = "1.31.0-alpha.2"
 authors = ["Apollo Graph, Inc. "]
 repository = "https://github.com/apollographql/router/"
 documentation = "https://docs.rs/apollo-router"
diff --git a/dockerfiles/tracing/docker-compose.datadog.yml b/dockerfiles/tracing/docker-compose.datadog.yml
index a3f7c92ada..3ca3720cb5 100644
--- a/dockerfiles/tracing/docker-compose.datadog.yml
+++ b/dockerfiles/tracing/docker-compose.datadog.yml
@@ -3,7 +3,7 @@ services:
 
   apollo-router:
     container_name: apollo-router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.1
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.2
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/datadog.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.jaeger.yml b/dockerfiles/tracing/docker-compose.jaeger.yml
index e3ae5c9f4f..8b047fc3c4 100644
--- a/dockerfiles/tracing/docker-compose.jaeger.yml
+++ b/dockerfiles/tracing/docker-compose.jaeger.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     #build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.1
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.2
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/jaeger.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.zipkin.yml b/dockerfiles/tracing/docker-compose.zipkin.yml
index 6db36f066c..795c17057a 100644
--- a/dockerfiles/tracing/docker-compose.zipkin.yml
+++ b/dockerfiles/tracing/docker-compose.zipkin.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.1
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.2
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/zipkin.router.yaml:/etc/config/configuration.yaml
diff --git a/docs/source/containerization/docker.mdx b/docs/source/containerization/docker.mdx
index 5440d1c56f..a8a42b7a81 100644
--- a/docs/source/containerization/docker.mdx
+++ b/docs/source/containerization/docker.mdx
@@ -11,7 +11,7 @@ The default behaviour of the router images is suitable for a quickstart or devel
 
 Note: The [docker documentation](https://docs.docker.com/engine/reference/run/) for the run command may be helpful when reading through the examples.
 
-Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.1`
+Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.2`
 
 ## Override the configuration
 
diff --git a/docs/source/containerization/kubernetes.mdx b/docs/source/containerization/kubernetes.mdx
index bda87c8353..4f4dc93a03 100644
--- a/docs/source/containerization/kubernetes.mdx
+++ b/docs/source/containerization/kubernetes.mdx
@@ -13,7 +13,7 @@ import { Link } from 'gatsby';
 
 [Helm](https://helm.sh) is the package manager for kubernetes.
 
-There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.1/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
+There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.2/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
 
 In both the following examples, we are using helm to install the router:
  - into namespace "router-deploy" (create namespace if it doesn't exist)
@@ -64,10 +64,10 @@ kind: ServiceAccount
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.1
+    helm.sh/chart: router-1.31.0-alpha.2
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.1"
+    app.kubernetes.io/version: "v1.31.0-alpha.2"
     app.kubernetes.io/managed-by: Helm
 ---
 # Source: router/templates/secret.yaml
@@ -76,10 +76,10 @@ kind: Secret
 metadata:
   name: "release-name-router"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.1
+    helm.sh/chart: router-1.31.0-alpha.2
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.1"
+    app.kubernetes.io/version: "v1.31.0-alpha.2"
     app.kubernetes.io/managed-by: Helm
 data:
   managedFederationApiKey: "UkVEQUNURUQ="
@@ -90,10 +90,10 @@ kind: ConfigMap
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.1
+    helm.sh/chart: router-1.31.0-alpha.2
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.1"
+    app.kubernetes.io/version: "v1.31.0-alpha.2"
     app.kubernetes.io/managed-by: Helm
 data:
   configuration.yaml: |
@@ -117,10 +117,10 @@ kind: Service
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.1
+    helm.sh/chart: router-1.31.0-alpha.2
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.1"
+    app.kubernetes.io/version: "v1.31.0-alpha.2"
     app.kubernetes.io/managed-by: Helm
 spec:
   type: ClusterIP
@@ -143,10 +143,10 @@ kind: Deployment
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.1
+    helm.sh/chart: router-1.31.0-alpha.2
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.1"
+    app.kubernetes.io/version: "v1.31.0-alpha.2"
     app.kubernetes.io/managed-by: Helm
   
   annotations:
@@ -174,7 +174,7 @@ spec:
         - name: router
           securityContext:
             {}
-          image: "ghcr.io/apollographql/router:v1.31.0-alpha.1"
+          image: "ghcr.io/apollographql/router:v1.31.0-alpha.2"
           imagePullPolicy: IfNotPresent
           args:
             - --hot-reload
@@ -226,10 +226,10 @@ kind: Pod
 metadata:
   name: "release-name-router-test-connection"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.1
+    helm.sh/chart: router-1.31.0-alpha.2
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.1"
+    app.kubernetes.io/version: "v1.31.0-alpha.2"
     app.kubernetes.io/managed-by: Helm
   annotations:
     "helm.sh/hook": test
diff --git a/helm/chart/router/Chart.yaml b/helm/chart/router/Chart.yaml
index a66745badc..34a12ac72a 100644
--- a/helm/chart/router/Chart.yaml
+++ b/helm/chart/router/Chart.yaml
@@ -20,10 +20,10 @@ type: application
 # so it matches the shape of our release process and release automation.
 # By proxy of that decision, this version uses SemVer 2.0.0, though the prefix
 # of "v" is not included.
-version: 1.31.0-alpha.1
+version: 1.31.0-alpha.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.31.0-alpha.1"
+appVersion: "v1.31.0-alpha.2"
diff --git a/helm/chart/router/README.md b/helm/chart/router/README.md
index 56de45829f..4328719c02 100644
--- a/helm/chart/router/README.md
+++ b/helm/chart/router/README.md
@@ -2,7 +2,7 @@
 
 [router](https://github.com/apollographql/router) Rust Graph Routing runtime for Apollo Federation
 
-![Version: 1.31.0-alpha.1](https://img.shields.io/badge/Version-1.31.0--alpha.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.1](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.1-informational?style=flat-square)
+![Version: 1.31.0-alpha.2](https://img.shields.io/badge/Version-1.31.0--alpha.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.2](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.2-informational?style=flat-square)
 
 ## Prerequisites
 
@@ -11,7 +11,7 @@
 ## Get Repo Info
 
 ```console
-helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.1
+helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.2
 ```
 
 ## Install Chart
@@ -19,7 +19,7 @@ helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.
 **Important:** only helm3 is supported
 
 ```console
-helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.1 --values my-values.yaml
+helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.2 --values my-values.yaml
 ```
 
 _See [configuration](#configuration) below._
diff --git a/scripts/install.sh b/scripts/install.sh
index 20aa4309f4..6d3c682c23 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -11,7 +11,7 @@ BINARY_DOWNLOAD_PREFIX="https://github.com/apollographql/router/releases/downloa
 
 # Router version defined in apollo-router's Cargo.toml
 # Note: Change this line manually during the release steps.
-PACKAGE_VERSION="v1.31.0-alpha.1"
+PACKAGE_VERSION="v1.31.0-alpha.2"
 
 download_binary() {
     downloader --check

From 7d45877d1bed8aa68f8e66a80f2317bc2ae9d998 Mon Sep 17 00:00:00 2001
From: Geoffroy Couprie 
Date: Mon, 25 Sep 2023 16:16:47 +0200
Subject: [PATCH 45/51] HTTP/2 Cleartext protocol (h2c) support for subgraph
 connections (#3852)

Fix #3535
The router can now connect to subgraphs over HTTP/2 Cleartext, which uses the HTTP/2 binary protocol directly over TCP without TLS. To activate it, set the `experimental_http2` option to `http2_only`.

---------

Co-authored-by: Edward Huang <18322228+shorgi@users.noreply.github.com>
Co-authored-by: Jeremy Lempereur 
---
 .changesets/feat_geal_h2c.md                  |   5 +
 apollo-router/src/configuration/metrics.rs    |   2 +-
 .../migrations/0011-experimental-http2.yaml   |  25 ++++
 ...nfiguration__tests__schema_generation.snap |  64 +++++++--
 ...guration__upgrade__test__change_field.snap |  14 ++
 .../metrics/traffic_shaping.router.yaml       |   2 +-
 apollo-router/src/configuration/upgrade.rs    |  32 +++++
 .../src/plugins/traffic_shaping/mod.rs        |  89 +++++++------
 .../src/services/subgraph_service.rs          | 125 ++++++++++++++----
 docs/source/configuration/traffic-shaping.mdx |   9 ++
 10 files changed, 293 insertions(+), 74 deletions(-)
 create mode 100644 .changesets/feat_geal_h2c.md
 create mode 100644 apollo-router/src/configuration/migrations/0011-experimental-http2.yaml
 create mode 100644 apollo-router/src/configuration/snapshots/apollo_router__configuration__upgrade__test__change_field.snap

diff --git a/.changesets/feat_geal_h2c.md b/.changesets/feat_geal_h2c.md
new file mode 100644
index 0000000000..0852c40671
--- /dev/null
+++ b/.changesets/feat_geal_h2c.md
@@ -0,0 +1,5 @@
+### HTTP/2 Cleartext protocol (h2c) support for subgraph connections ([Issue #3535](https://github.com/apollographql/router/issues/3535))
+
+The router can now connect to subgraphs over HTTP/2 Cleartext, which uses the HTTP/2 binary protocol directly over TCP without TLS. To activate it, set the `experimental_http2` option to `http2_only`.
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3852
\ No newline at end of file
diff --git a/apollo-router/src/configuration/metrics.rs b/apollo-router/src/configuration/metrics.rs
index 20dd595598..9506b1e1f2 100644
--- a/apollo-router/src/configuration/metrics.rs
+++ b/apollo-router/src/configuration/metrics.rs
@@ -282,7 +282,7 @@ impl Metrics {
             opt.subgraph.rate_limit,
             "$[?(@.all.global_rate_limit || @.subgraphs..global_rate_limit)]",
             opt.subgraph.http2,
-            "$[?(@.all.experimental_enable_http2 == true || @.subgraphs..experimental_enable_http2 == true)]",
+            "$[?(@.all.experimental_http2 == 'enable' || @.all.experimental_http2 == 'http2only' || @.subgraphs..experimental_http2 == 'enable' || @.subgraphs..experimental_http2 == 'http2only')]",
             opt.subgraph.compression,
             "$[?(@.all.compression || @.subgraphs..compression)]",
             opt.subgraph.deduplicate_query,
diff --git a/apollo-router/src/configuration/migrations/0011-experimental-http2.yaml b/apollo-router/src/configuration/migrations/0011-experimental-http2.yaml
new file mode 100644
index 0000000000..e73b228a03
--- /dev/null
+++ b/apollo-router/src/configuration/migrations/0011-experimental-http2.yaml
@@ -0,0 +1,25 @@
+description: Move experimental_enable_http2 to experimental_http2
+actions:
+  - type: change
+    path: traffic_shaping.all.experimental_enable_http2
+    from: true
+    to: enable
+  - type: change
+    path: traffic_shaping.all.experimental_enable_http2
+    from: false
+    to: disable
+  - type: move
+    from: traffic_shaping.all.experimental_enable_http2
+    to: traffic_shaping.all.experimental_http2
+
+  - type: change
+    path: traffic_shaping.subgraphs..experimental_enable_http2
+    from: true
+    to: enable
+  - type: change
+    path: traffic_shaping.subgraphs..experimental_enable_http2
+    from: false
+    to: disable
+  - type: move
+    from: traffic_shaping.subgraphs..experimental_enable_http2
+    to: traffic_shaping.subgraphs..experimental_http2
diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap
index 428ef466d4..20fba9012f 100644
--- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap
+++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap
@@ -5558,11 +5558,6 @@ expression: "&schema"
               "type": "boolean",
               "nullable": true
             },
-            "experimental_enable_http2": {
-              "description": "Enable HTTP2 for subgraphs",
-              "type": "boolean",
-              "nullable": true
-            },
             "experimental_entity_caching": {
               "description": "Enable entity caching",
               "type": "object",
@@ -5578,6 +5573,33 @@ expression: "&schema"
               "additionalProperties": false,
               "nullable": true
             },
+            "experimental_http2": {
+              "description": "Enable HTTP2 for subgraphs",
+              "oneOf": [
+                {
+                  "description": "Enable HTTP2 for subgraphs",
+                  "type": "string",
+                  "enum": [
+                    "enable"
+                  ]
+                },
+                {
+                  "description": "Disable HTTP2 for subgraphs",
+                  "type": "string",
+                  "enum": [
+                    "disable"
+                  ]
+                },
+                {
+                  "description": "Only HTTP2 is active",
+                  "type": "string",
+                  "enum": [
+                    "http2only"
+                  ]
+                }
+              ],
+              "nullable": true
+            },
             "experimental_retry": {
               "description": "Retry configuration",
               "type": "object",
@@ -5746,11 +5768,6 @@ expression: "&schema"
                 "type": "boolean",
                 "nullable": true
               },
-              "experimental_enable_http2": {
-                "description": "Enable HTTP2 for subgraphs",
-                "type": "boolean",
-                "nullable": true
-              },
               "experimental_entity_caching": {
                 "description": "Enable entity caching",
                 "type": "object",
@@ -5766,6 +5783,33 @@ expression: "&schema"
                 "additionalProperties": false,
                 "nullable": true
               },
+              "experimental_http2": {
+                "description": "Enable HTTP2 for subgraphs",
+                "oneOf": [
+                  {
+                    "description": "Enable HTTP2 for subgraphs",
+                    "type": "string",
+                    "enum": [
+                      "enable"
+                    ]
+                  },
+                  {
+                    "description": "Disable HTTP2 for subgraphs",
+                    "type": "string",
+                    "enum": [
+                      "disable"
+                    ]
+                  },
+                  {
+                    "description": "Only HTTP2 is active",
+                    "type": "string",
+                    "enum": [
+                      "http2only"
+                    ]
+                  }
+                ],
+                "nullable": true
+              },
               "experimental_retry": {
                 "description": "Retry configuration",
                 "type": "object",
diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__upgrade__test__change_field.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__upgrade__test__change_field.snap
new file mode 100644
index 0000000000..d1a8ad8ae0
--- /dev/null
+++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__upgrade__test__change_field.snap
@@ -0,0 +1,14 @@
+---
+source: apollo-router/src/configuration/upgrade.rs
+expression: "apply_migration(&source_doc(),\n        &Migration::builder().action(Action::Change {\n                            path: \"obj.field1\".to_string(),\n                            from: Value::Number(1u64.into()),\n                            to: Value::String(\"a\".into()),\n                        }).description(\"change field1\").build()).expect(\"expected successful migration\")"
+---
+{
+  "obj": {
+    "field1": "a",
+    "field2": 2
+  },
+  "arr": [
+    "v1",
+    "v2"
+  ]
+}
diff --git a/apollo-router/src/configuration/testdata/metrics/traffic_shaping.router.yaml b/apollo-router/src/configuration/testdata/metrics/traffic_shaping.router.yaml
index b5304812f4..ad221ae92d 100644
--- a/apollo-router/src/configuration/testdata/metrics/traffic_shaping.router.yaml
+++ b/apollo-router/src/configuration/testdata/metrics/traffic_shaping.router.yaml
@@ -11,7 +11,7 @@ traffic_shaping:
     global_rate_limit:
       capacity: 100
       interval: 1s
-    experimental_enable_http2: true
+    experimental_http2: enable
     experimental_retry:
       ttl: 1s
       min_per_sec: 2
diff --git a/apollo-router/src/configuration/upgrade.rs b/apollo-router/src/configuration/upgrade.rs
index c88e2960dc..11d6fee51b 100644
--- a/apollo-router/src/configuration/upgrade.rs
+++ b/apollo-router/src/configuration/upgrade.rs
@@ -38,6 +38,11 @@ enum Action {
         from: String,
         to: String,
     },
+    Change {
+        path: String,
+        from: Value,
+        to: Value,
+    },
 }
 
 const REMOVAL_VALUE: &str = "__PLEASE_DELETE_ME";
@@ -129,6 +134,17 @@ fn apply_migration(config: &Value, migration: &Migration) -> Result {
+                if !jsonpath_lib::select(config, &format!("$.{path} == {from}"))
+                    .unwrap_or_default()
+                    .is_empty()
+                {
+                    transformer_builder = transformer_builder.add_action(
+                        Parser::parse(&format!(r#"const({to})"#), path)
+                            .expect("migration must be valid"),
+                    );
+                }
+            }
         }
     }
     let transformer = transformer_builder
@@ -423,4 +439,20 @@ mod test {
         )
         .expect("expected successful migration"));
     }
+
+    #[test]
+    fn change_field() {
+        insta::assert_json_snapshot!(apply_migration(
+            &source_doc(),
+            &Migration::builder()
+                .action(Action::Change {
+                    path: "obj.field1".to_string(),
+                    from: Value::Number(1u64.into()),
+                    to: Value::String("a".into()),
+                })
+                .description("change field1")
+                .build(),
+        )
+        .expect("expected successful migration"));
+    }
 }
diff --git a/apollo-router/src/plugins/traffic_shaping/mod.rs b/apollo-router/src/plugins/traffic_shaping/mod.rs
index 6d21338d73..566534b82d 100644
--- a/apollo-router/src/plugins/traffic_shaping/mod.rs
+++ b/apollo-router/src/plugins/traffic_shaping/mod.rs
@@ -73,7 +73,19 @@ struct Shaping {
     //  *experimental feature*: Enables request retry
     experimental_retry: Option,
     /// Enable HTTP2 for subgraphs
-    experimental_enable_http2: Option,
+    experimental_http2: Option,
+}
+
+#[derive(PartialEq, Default, Debug, Clone, Deserialize, JsonSchema)]
+#[serde(rename_all = "lowercase")]
+pub(crate) enum Http2Config {
+    #[default]
+    /// Enable HTTP2 for subgraphs
+    Enable,
+    /// Disable HTTP2 for subgraphs
+    Disable,
+    /// Only HTTP2 is active
+    Http2Only,
 }
 
 impl Merge for Shaping {
@@ -94,10 +106,10 @@ impl Merge for Shaping {
                     .as_ref()
                     .or(fallback.experimental_retry.as_ref())
                     .cloned(),
-                experimental_enable_http2: self
-                    .experimental_enable_http2
+                experimental_http2: self
+                    .experimental_http2
                     .as_ref()
-                    .or(fallback.experimental_enable_http2.as_ref())
+                    .or(fallback.experimental_http2.as_ref())
                     .cloned(),
             },
         }
@@ -458,13 +470,13 @@ impl TrafficShaping {
         }
     }
 
-    pub(crate) fn enable_subgraph_http2(&self, service_name: &str) -> bool {
+    pub(crate) fn enable_subgraph_http2(&self, service_name: &str) -> Http2Config {
         Self::merge_config(
             self.config.all.as_ref(),
             self.config.subgraphs.get(service_name),
         )
-        .and_then(|config| config.shaping.experimental_enable_http2)
-        .unwrap_or(true)
+        .and_then(|config| config.shaping.experimental_http2)
+        .unwrap_or(Http2Config::Enable)
     }
 }
 
@@ -714,39 +726,42 @@ mod test {
         let config = serde_yaml::from_str::(
             r#"
         all:
-          experimental_enable_http2: false
+          experimental_http2: disable
         subgraphs: 
           products:
-            experimental_enable_http2: true
+            experimental_http2: enable
           reviews:
-            experimental_enable_http2: false
+            experimental_http2: disable
         router:
           timeout: 65s
         "#,
         )
         .unwrap();
 
-        assert!(TrafficShaping::merge_config(
-            config.all.as_ref(),
-            config.subgraphs.get("products")
-        )
-        .unwrap()
-        .shaping
-        .experimental_enable_http2
-        .unwrap());
-        assert!(!TrafficShaping::merge_config(
-            config.all.as_ref(),
-            config.subgraphs.get("reviews")
-        )
-        .unwrap()
-        .shaping
-        .experimental_enable_http2
-        .unwrap());
-        assert!(!TrafficShaping::merge_config(config.all.as_ref(), None)
-            .unwrap()
-            .shaping
-            .experimental_enable_http2
-            .unwrap());
+        assert!(
+            TrafficShaping::merge_config(config.all.as_ref(), config.subgraphs.get("products"))
+                .unwrap()
+                .shaping
+                .experimental_http2
+                .unwrap()
+                == Http2Config::Enable
+        );
+        assert!(
+            TrafficShaping::merge_config(config.all.as_ref(), config.subgraphs.get("reviews"))
+                .unwrap()
+                .shaping
+                .experimental_http2
+                .unwrap()
+                == Http2Config::Disable
+        );
+        assert!(
+            TrafficShaping::merge_config(config.all.as_ref(), None)
+                .unwrap()
+                .shaping
+                .experimental_http2
+                .unwrap()
+                == Http2Config::Disable
+        );
     }
 
     #[tokio::test]
@@ -754,12 +769,12 @@ mod test {
         let config = serde_yaml::from_str::(
             r#"
         all:
-          experimental_enable_http2: false
+          experimental_http2: disable
         subgraphs: 
           products:
-            experimental_enable_http2: true
+            experimental_http2: enable
           reviews:
-            experimental_enable_http2: false
+            experimental_http2: disable
         router:
           timeout: 65s
         "#,
@@ -770,9 +785,9 @@ mod test {
             .await
             .unwrap();
 
-        assert!(shaping_config.enable_subgraph_http2("products"));
-        assert!(!shaping_config.enable_subgraph_http2("reviews"));
-        assert!(!shaping_config.enable_subgraph_http2("this_doesnt_exist"));
+        assert!(shaping_config.enable_subgraph_http2("products") == Http2Config::Enable);
+        assert!(shaping_config.enable_subgraph_http2("reviews") == Http2Config::Disable);
+        assert!(shaping_config.enable_subgraph_http2("this_doesnt_exist") == Http2Config::Disable);
     }
 
     #[tokio::test(flavor = "multi_thread")]
diff --git a/apollo-router/src/services/subgraph_service.rs b/apollo-router/src/services/subgraph_service.rs
index 78e870bf17..b8b2e39591 100644
--- a/apollo-router/src/services/subgraph_service.rs
+++ b/apollo-router/src/services/subgraph_service.rs
@@ -70,6 +70,7 @@ use crate::plugins::subscription::WebSocketConfiguration;
 use crate::plugins::subscription::SUBSCRIPTION_WS_CUSTOM_CONNECTION_PARAMS;
 use crate::plugins::telemetry::LOGGING_DISPLAY_BODY;
 use crate::plugins::telemetry::LOGGING_DISPLAY_HEADERS;
+use crate::plugins::traffic_shaping::Http2Config;
 use crate::protocols::websocket::convert_websocket_stream;
 use crate::protocols::websocket::GraphqlWebSocket;
 use crate::query_planner::OperationKind;
@@ -160,7 +161,7 @@ impl SubgraphService {
         service: impl Into,
         configuration: &Configuration,
         tls_root_store: &Option,
-        enable_http2: bool,
+        http2: Http2Config,
         subscription_config: Option,
     ) -> Result {
         let name: String = service.into();
@@ -217,7 +218,7 @@ impl SubgraphService {
         Ok(SubgraphService::new(
             name,
             enable_apq,
-            enable_http2,
+            http2,
             subscription_config,
             tls_client_config,
             configuration.notify.clone(),
@@ -227,7 +228,7 @@ impl SubgraphService {
     pub(crate) fn new(
         service: impl Into,
         enable_apq: bool,
-        enable_http2: bool,
+        http2: Http2Config,
         subscription_config: Option,
         tls_config: ClientConfig,
         notify: Notify,
@@ -242,7 +243,7 @@ impl SubgraphService {
             .https_or_http()
             .enable_http1();
 
-        let connector = if enable_http2 {
+        let connector = if http2 != Http2Config::Disable {
             builder.enable_http2().wrap_connector(http_connector)
         } else {
             builder.wrap_connector(http_connector)
@@ -250,6 +251,7 @@ impl SubgraphService {
 
         let http_client = hyper::Client::builder()
             .pool_idle_timeout(POOL_IDLE_TIMEOUT_DURATION)
+            .http2_only(http2 == Http2Config::Http2Only)
             .build(connector);
         Self {
             client: ServiceBuilder::new()
@@ -1850,7 +1852,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "testbis",
             true,
-            false,
+            Http2Config::Disable,
             subscription_config().into(),
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -1905,7 +1907,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -1948,7 +1950,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -1991,7 +1993,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2039,7 +2041,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2091,7 +2093,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2141,7 +2143,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            false,
+            Http2Config::Disable,
             subscription_config().into(),
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2204,7 +2206,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            false,
+            Http2Config::Disable,
             subscription_config().into(),
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2259,7 +2261,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2310,7 +2312,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2356,7 +2358,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             false,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2406,7 +2408,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2452,7 +2454,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2507,7 +2509,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2560,7 +2562,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2610,7 +2612,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2660,7 +2662,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             true,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2710,7 +2712,7 @@ mod tests {
         let subgraph_service = SubgraphService::new(
             "test",
             false,
-            true,
+            Http2Config::Enable,
             None,
             ClientConfig::builder()
                 .with_safe_defaults()
@@ -2803,7 +2805,8 @@ mod tests {
             },
         );
         let subgraph_service =
-            SubgraphService::from_config("test", &config, &None, false, None).unwrap();
+            SubgraphService::from_config("test", &config, &None, Http2Config::Enable, None)
+                .unwrap();
 
         let url = Uri::from_str(&format!("https://localhost:{}", socket_addr.port())).unwrap();
         let response = subgraph_service
@@ -2828,6 +2831,7 @@ mod tests {
             })
             .await
             .unwrap();
+
         assert_eq!(response.response.body().data, Some(Value::Null));
     }
 
@@ -2857,7 +2861,8 @@ mod tests {
             },
         );
         let subgraph_service =
-            SubgraphService::from_config("test", &config, &None, false, None).unwrap();
+            SubgraphService::from_config("test", &config, &None, Http2Config::Enable, None)
+                .unwrap();
 
         let url = Uri::from_str(&format!("https://localhost:{}", socket_addr.port())).unwrap();
         let response = subgraph_service
@@ -2965,7 +2970,8 @@ mod tests {
             },
         );
         let subgraph_service =
-            SubgraphService::from_config("test", &config, &None, false, None).unwrap();
+            SubgraphService::from_config("test", &config, &None, Http2Config::Enable, None)
+                .unwrap();
 
         let url = Uri::from_str(&format!("https://localhost:{}", socket_addr.port())).unwrap();
         let response = subgraph_service
@@ -2992,4 +2998,73 @@ mod tests {
             .unwrap();
         assert_eq!(response.response.body().data, Some(Value::Null));
     }
+
+    // starts a local server emulating a subgraph returning status code 401
+    async fn emulate_h2c_server(listener: TcpListener) {
+        async fn handle(_request: http::Request) -> Result, Infallible> {
+            println!("h2C server got req: {_request:?}");
+            Ok(http::Response::builder()
+                .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                .status(StatusCode::OK)
+                .body(
+                    serde_json::to_string(&Response {
+                        data: Some(Value::default()),
+                        ..Response::default()
+                    })
+                    .expect("always valid")
+                    .into(),
+                )
+                .unwrap())
+        }
+
+        let make_svc = make_service_fn(|_conn| async { Ok::<_, Infallible>(service_fn(handle)) });
+        let server = Server::from_tcp(listener)
+            .unwrap()
+            .http2_only(true)
+            .serve(make_svc);
+        server.await.unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_subgraph_h2c() {
+        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+        let socket_addr = listener.local_addr().unwrap();
+        tokio::task::spawn(emulate_h2c_server(listener));
+        let subgraph_service = SubgraphService::new(
+            "test",
+            true,
+            Http2Config::Http2Only,
+            None,
+            rustls::ClientConfig::builder()
+                .with_safe_defaults()
+                .with_native_roots()
+                .with_no_client_auth(),
+            Notify::default(),
+        );
+
+        let url = Uri::from_str(&format!("http://{socket_addr}")).unwrap();
+        let response = subgraph_service
+            .oneshot(SubgraphRequest {
+                supergraph_request: Arc::new(
+                    http::Request::builder()
+                        .header(HOST, "host")
+                        .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                        .body(Request::builder().query("query").build())
+                        .expect("expecting valid request"),
+                ),
+                subgraph_request: http::Request::builder()
+                    .header(HOST, "rhost")
+                    .header(CONTENT_TYPE, APPLICATION_JSON.essence_str())
+                    .uri(url)
+                    .body(Request::builder().query("query").build())
+                    .expect("expecting valid request"),
+                operation_kind: OperationKind::Query,
+                context: Context::new(),
+                subscription_stream: None,
+                connection_closed_signal: None,
+            })
+            .await
+            .unwrap();
+        assert!(response.response.body().errors.is_empty());
+    }
 }
diff --git a/docs/source/configuration/traffic-shaping.mdx b/docs/source/configuration/traffic-shaping.mdx
index b16723690e..52b428781b 100644
--- a/docs/source/configuration/traffic-shaping.mdx
+++ b/docs/source/configuration/traffic-shaping.mdx
@@ -32,6 +32,7 @@ traffic_shaping:
         ttl: 10s # for each successful request, we register a token, that expires according to this option (default: 10s)
         retry_percent: 0.2 # defines the proportion of available retries to the current number of tokens
         retry_mutations: false # allows retries on mutations. This should only be enabled if mutations are idempotent
+      experimental_http2: enable # Configures HTTP/2 usage. Can be 'enable' (default), 'disable' or 'http2only'
 ```
 
 ## Client side traffic shaping
@@ -151,6 +152,14 @@ traffic_shaping:
     deduplicate_query: true # Enable query deduplication for all subgraphs.
 ```
 
+### HTTP/2
+
+The router supports subgraph connections over
+- HTTP/2 with TLS. This is the default configuration.
+- HTTP/2 Cleartext protocol (h2c). This uses HTTP/2 over plaintext connections.
+
+To use h2c, the subgraph URL must have the `http` scheme, and the `experimental_http2` option must be set to `http2only`.
+
 ### Ordering
 
 Traffic shaping always executes these steps in the same order, to ensure a consistent behaviour. Declaration order in the configuration will not affect the runtime order:

From 18db388a09718d86f2c5b85d5bdde9cd3fd9c73e Mon Sep 17 00:00:00 2001
From: Simon Sapin 
Date: Mon, 25 Sep 2023 18:40:52 +0200
Subject: [PATCH 46/51] Fix validation error with ID variable values
 overflowing i32 (#3896)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #3873

Input values for variables of type `ID` were previously validated as
"either like a GraphQL `Int` or like a GraphQL `String`". GraphQL `Int`
is specified as a signed 32-bit integer, such that values that overflow
fail validation. Applying this range restriction to `ID` values was
incorrect. Instead, validation for `ID` now accepts any JSON integer or
JSON string value, so that IDs larger than 32 bits can be used.


---

**Checklist**

Complete the checklist (and note appropriate exceptions) before the PR
is marked ready-for-review.

- [x] Changes are compatible[^1]
- [x] Documentation[^2] completed
- [x] Performance impact assessed and acceptable
- Tests added and passing[^3]
    - [ ] Unit Tests
    - [x] Integration Tests
    - [ ] Manual Tests

**Exceptions**

*Note any exceptions here*

**Notes**

[^1]: It may be appropriate to bring upcoming changes to the attention
of other (impacted) groups. Please endeavour to do this before seeking
PR approval. The mechanism for doing this will vary considerably, so use
your judgement as to how and when to do this.
[^2]: Configuration is an important part of many changes. Where
applicable please try to document configuration examples.
[^3]: Tick whichever testing boxes are applicable. If you are adding
Manual Tests, please document the manual testing (extensively) in the
Exceptions.
---
 .changesets/fix_simon_large_id.md             |  5 ++
 apollo-router/src/json_ext.rs                 | 14 +++++
 .../src/services/supergraph_service.rs        | 57 +++++++++++++++++++
 apollo-router/src/spec/field_type.rs          |  4 +-
 4 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 .changesets/fix_simon_large_id.md

diff --git a/.changesets/fix_simon_large_id.md b/.changesets/fix_simon_large_id.md
new file mode 100644
index 0000000000..58fdfcdebe
--- /dev/null
+++ b/.changesets/fix_simon_large_id.md
@@ -0,0 +1,5 @@
+### Fix validation error with ID variable values overflowing i32 ([Issue #3873](https://github.com/apollographql/router/issues/3873))
+
+Input values for variables of type `ID` were previously validated as "either like a GraphQL `Int` or like a GraphQL `String`". GraphQL `Int` is specified as a signed 32-bit integer, such that values that overflow fail validation. Applying this range restriction to `ID` values was incorrect. Instead, validation for `ID` now accepts any JSON integer or JSON string value, so that IDs larger than 32 bits can be used.
+
+By [@SimonSapin](https://github.com/SimonSapin) in https://github.com/apollographql/router/pull/3896
diff --git a/apollo-router/src/json_ext.rs b/apollo-router/src/json_ext.rs
index 6955f15924..3c7d7bd07f 100644
--- a/apollo-router/src/json_ext.rs
+++ b/apollo-router/src/json_ext.rs
@@ -103,6 +103,9 @@ pub(crate) trait ValueExt {
     #[track_caller]
     fn is_valid_int_input(&self) -> bool;
 
+    #[track_caller]
+    fn is_valid_id_input(&self) -> bool;
+
     /// Returns whether this value is an object that matches the provided type.
     ///
     /// More precisely, this checks that this value is an object, looks at
@@ -394,6 +397,17 @@ impl ValueExt for Value {
         iterate_path_mut(schema, &mut Path::default(), &path.0, self, &mut f)
     }
 
+    #[track_caller]
+    fn is_valid_id_input(&self) -> bool {
+        // https://spec.graphql.org/October2021/#sec-ID.Input-Coercion
+        match self {
+            // Any string and integer values are accepted
+            Value::String(_) => true,
+            Value::Number(n) => n.is_i64() || n.is_u64(),
+            _ => false,
+        }
+    }
+
     #[track_caller]
     fn is_valid_float_input(&self) -> bool {
         // https://spec.graphql.org/draft/#sec-Float.Input-Coercion
diff --git a/apollo-router/src/services/supergraph_service.rs b/apollo-router/src/services/supergraph_service.rs
index 5bf5ce0159..207605ef55 100644
--- a/apollo-router/src/services/supergraph_service.rs
+++ b/apollo-router/src/services/supergraph_service.rs
@@ -778,8 +778,11 @@ mod tests {
     use std::collections::HashMap;
     use std::time::Duration;
 
+    use tower::ServiceExt;
+
     use super::*;
     use crate::plugin::test::MockSubgraph;
+    use crate::services::subgraph;
     use crate::services::supergraph;
     use crate::test_harness::MockedSubgraphs;
     use crate::Notify;
@@ -3308,4 +3311,58 @@ mod tests {
         let response = stream.next_response().await.unwrap();
         assert_eq!(serde_json_bytes::Value::Null, response.data.unwrap());
     }
+
+    #[tokio::test]
+    async fn id_scalar_can_overflow_i32() {
+        // Hack to let the first subgraph fetch contain an ID variable:
+        // ```
+        // type Query {
+        //     user(id: ID!): User @join__field(graph: USER)
+        // }
+        // ```
+        assert!(SCHEMA.contains("currentUser:"));
+        let schema = SCHEMA.replace("currentUser:", "user(id: ID!):");
+
+        let service = TestHarness::builder()
+            .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } }))
+            .unwrap()
+            .schema(&schema)
+            .subgraph_hook(|_subgraph_name, _service| {
+                tower::service_fn(|request: subgraph::Request| async move {
+                    let id = &request.subgraph_request.body().variables["id"];
+                    Err(format!("$id = {id}").into())
+                })
+                .boxed()
+            })
+            .build_supergraph()
+            .await
+            .unwrap();
+
+        let large: i64 = 1 << 53;
+        let large_plus_one = large + 1;
+        // f64 rounds since it doesn’t have enough mantissa bits
+        assert!(large_plus_one as f64 as i64 == large);
+        // i64 of course doesn’t round
+        assert!(large_plus_one != large);
+
+        let request = supergraph::Request::fake_builder()
+            .query("query($id: ID!) { user(id: $id) { name }}")
+            .variable("id", large_plus_one)
+            .build()
+            .unwrap();
+        let response = service
+            .oneshot(request)
+            .await
+            .unwrap()
+            .next_response()
+            .await
+            .unwrap();
+        // The router did not panic or respond with an early validation error.
+        // Instead it did a subgraph fetch, which recieved the correct ID variable without rounding:
+        assert_eq!(
+            response.errors[0].extensions["reason"].as_str().unwrap(),
+            "$id = 9007199254740993"
+        );
+        assert_eq!(large_plus_one.to_string(), "9007199254740993");
+    }
 }
diff --git a/apollo-router/src/spec/field_type.rs b/apollo-router/src/spec/field_type.rs
index a08bda2652..529571126a 100644
--- a/apollo-router/src/spec/field_type.rs
+++ b/apollo-router/src/spec/field_type.rs
@@ -109,8 +109,8 @@ fn validate_input_value(
         //
         // In practice it seems Int works too
         (hir::Type::Named { name, .. }, Value::String(_)) if name == "ID" => Ok(()),
-        (hir::Type::Named { name, .. }, maybe_int) if name == "ID" => {
-            if maybe_int == &Value::Null || maybe_int.is_valid_int_input() {
+        (hir::Type::Named { name, .. }, value) if name == "ID" => {
+            if value == &Value::Null || value.is_valid_id_input() {
                 Ok(())
             } else {
                 Err(InvalidValue)

From 76fb4f0ddec49a3c89cc1960af9a3ac3920dde59 Mon Sep 17 00:00:00 2001
From: Jesse Rosenberger 
Date: Mon, 25 Sep 2023 20:21:35 +0000
Subject: [PATCH 47/51] prep release: v1.31.0-alpha.3

---
 Cargo.lock                                    |  6 ++--
 apollo-router-benchmarks/Cargo.toml           |  2 +-
 apollo-router-scaffold/Cargo.toml             |  2 +-
 .../templates/base/Cargo.toml                 |  2 +-
 .../templates/base/xtask/Cargo.toml           |  2 +-
 apollo-router/Cargo.toml                      |  2 +-
 .../tracing/docker-compose.datadog.yml        |  2 +-
 dockerfiles/tracing/docker-compose.jaeger.yml |  2 +-
 dockerfiles/tracing/docker-compose.zipkin.yml |  2 +-
 docs/source/containerization/docker.mdx       |  2 +-
 docs/source/containerization/kubernetes.mdx   | 28 +++++++++----------
 helm/chart/router/Chart.yaml                  |  4 +--
 helm/chart/router/README.md                   |  6 ++--
 scripts/install.sh                            |  2 +-
 14 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index a681a7bbb6..96052fc58e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -264,7 +264,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router"
-version = "1.31.0-alpha.2"
+version = "1.31.0-alpha.3"
 dependencies = [
  "access-json",
  "anyhow",
@@ -411,7 +411,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.2"
+version = "1.31.0-alpha.3"
 dependencies = [
  "apollo-parser 0.6.2",
  "apollo-router",
@@ -427,7 +427,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.2"
+version = "1.31.0-alpha.3"
 dependencies = [
  "anyhow",
  "cargo-scaffold",
diff --git a/apollo-router-benchmarks/Cargo.toml b/apollo-router-benchmarks/Cargo.toml
index 1b12fa92cf..f68de3e70e 100644
--- a/apollo-router-benchmarks/Cargo.toml
+++ b/apollo-router-benchmarks/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.2"
+version = "1.31.0-alpha.3"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml
index 47ac91e956..2e058aced7 100644
--- a/apollo-router-scaffold/Cargo.toml
+++ b/apollo-router-scaffold/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.2"
+version = "1.31.0-alpha.3"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/templates/base/Cargo.toml b/apollo-router-scaffold/templates/base/Cargo.toml
index aab266039c..b26035f57c 100644
--- a/apollo-router-scaffold/templates/base/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/Cargo.toml
@@ -22,7 +22,7 @@ apollo-router = { path ="{{integration_test}}apollo-router" }
 apollo-router = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
 # Note if you update these dependencies then also update xtask/Cargo.toml
-apollo-router = "1.31.0-alpha.2"
+apollo-router = "1.31.0-alpha.3"
 {{/if}}
 {{/if}}
 async-trait = "0.1.52"
diff --git a/apollo-router-scaffold/templates/base/xtask/Cargo.toml b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
index 9c6341051d..376844c6ec 100644
--- a/apollo-router-scaffold/templates/base/xtask/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
@@ -13,7 +13,7 @@ apollo-router-scaffold = { path ="{{integration_test}}apollo-router-scaffold" }
 {{#if branch}}
 apollo-router-scaffold = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
-apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.2" }
+apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.3" }
 {{/if}}
 {{/if}}
 anyhow = "1.0.58"
diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml
index 1aeb8de459..df34444c55 100644
--- a/apollo-router/Cargo.toml
+++ b/apollo-router/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router"
-version = "1.31.0-alpha.2"
+version = "1.31.0-alpha.3"
 authors = ["Apollo Graph, Inc. "]
 repository = "https://github.com/apollographql/router/"
 documentation = "https://docs.rs/apollo-router"
diff --git a/dockerfiles/tracing/docker-compose.datadog.yml b/dockerfiles/tracing/docker-compose.datadog.yml
index 3ca3720cb5..f1a21c848e 100644
--- a/dockerfiles/tracing/docker-compose.datadog.yml
+++ b/dockerfiles/tracing/docker-compose.datadog.yml
@@ -3,7 +3,7 @@ services:
 
   apollo-router:
     container_name: apollo-router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.2
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.3
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/datadog.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.jaeger.yml b/dockerfiles/tracing/docker-compose.jaeger.yml
index 8b047fc3c4..25456fca33 100644
--- a/dockerfiles/tracing/docker-compose.jaeger.yml
+++ b/dockerfiles/tracing/docker-compose.jaeger.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     #build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.2
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.3
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/jaeger.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.zipkin.yml b/dockerfiles/tracing/docker-compose.zipkin.yml
index 795c17057a..857e30a98f 100644
--- a/dockerfiles/tracing/docker-compose.zipkin.yml
+++ b/dockerfiles/tracing/docker-compose.zipkin.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.2
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.3
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/zipkin.router.yaml:/etc/config/configuration.yaml
diff --git a/docs/source/containerization/docker.mdx b/docs/source/containerization/docker.mdx
index a8a42b7a81..31c10f9224 100644
--- a/docs/source/containerization/docker.mdx
+++ b/docs/source/containerization/docker.mdx
@@ -11,7 +11,7 @@ The default behaviour of the router images is suitable for a quickstart or devel
 
 Note: The [docker documentation](https://docs.docker.com/engine/reference/run/) for the run command may be helpful when reading through the examples.
 
-Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.2`
+Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.3`
 
 ## Override the configuration
 
diff --git a/docs/source/containerization/kubernetes.mdx b/docs/source/containerization/kubernetes.mdx
index 4f4dc93a03..072b71cbc8 100644
--- a/docs/source/containerization/kubernetes.mdx
+++ b/docs/source/containerization/kubernetes.mdx
@@ -13,7 +13,7 @@ import { Link } from 'gatsby';
 
 [Helm](https://helm.sh) is the package manager for kubernetes.
 
-There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.2/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
+There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.3/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
 
 In both the following examples, we are using helm to install the router:
  - into namespace "router-deploy" (create namespace if it doesn't exist)
@@ -64,10 +64,10 @@ kind: ServiceAccount
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.2
+    helm.sh/chart: router-1.31.0-alpha.3
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.2"
+    app.kubernetes.io/version: "v1.31.0-alpha.3"
     app.kubernetes.io/managed-by: Helm
 ---
 # Source: router/templates/secret.yaml
@@ -76,10 +76,10 @@ kind: Secret
 metadata:
   name: "release-name-router"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.2
+    helm.sh/chart: router-1.31.0-alpha.3
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.2"
+    app.kubernetes.io/version: "v1.31.0-alpha.3"
     app.kubernetes.io/managed-by: Helm
 data:
   managedFederationApiKey: "UkVEQUNURUQ="
@@ -90,10 +90,10 @@ kind: ConfigMap
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.2
+    helm.sh/chart: router-1.31.0-alpha.3
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.2"
+    app.kubernetes.io/version: "v1.31.0-alpha.3"
     app.kubernetes.io/managed-by: Helm
 data:
   configuration.yaml: |
@@ -117,10 +117,10 @@ kind: Service
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.2
+    helm.sh/chart: router-1.31.0-alpha.3
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.2"
+    app.kubernetes.io/version: "v1.31.0-alpha.3"
     app.kubernetes.io/managed-by: Helm
 spec:
   type: ClusterIP
@@ -143,10 +143,10 @@ kind: Deployment
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.2
+    helm.sh/chart: router-1.31.0-alpha.3
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.2"
+    app.kubernetes.io/version: "v1.31.0-alpha.3"
     app.kubernetes.io/managed-by: Helm
   
   annotations:
@@ -174,7 +174,7 @@ spec:
         - name: router
           securityContext:
             {}
-          image: "ghcr.io/apollographql/router:v1.31.0-alpha.2"
+          image: "ghcr.io/apollographql/router:v1.31.0-alpha.3"
           imagePullPolicy: IfNotPresent
           args:
             - --hot-reload
@@ -226,10 +226,10 @@ kind: Pod
 metadata:
   name: "release-name-router-test-connection"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.2
+    helm.sh/chart: router-1.31.0-alpha.3
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.2"
+    app.kubernetes.io/version: "v1.31.0-alpha.3"
     app.kubernetes.io/managed-by: Helm
   annotations:
     "helm.sh/hook": test
diff --git a/helm/chart/router/Chart.yaml b/helm/chart/router/Chart.yaml
index 34a12ac72a..14cb1a852d 100644
--- a/helm/chart/router/Chart.yaml
+++ b/helm/chart/router/Chart.yaml
@@ -20,10 +20,10 @@ type: application
 # so it matches the shape of our release process and release automation.
 # By proxy of that decision, this version uses SemVer 2.0.0, though the prefix
 # of "v" is not included.
-version: 1.31.0-alpha.2
+version: 1.31.0-alpha.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.31.0-alpha.2"
+appVersion: "v1.31.0-alpha.3"
diff --git a/helm/chart/router/README.md b/helm/chart/router/README.md
index 4328719c02..eef24017d4 100644
--- a/helm/chart/router/README.md
+++ b/helm/chart/router/README.md
@@ -2,7 +2,7 @@
 
 [router](https://github.com/apollographql/router) Rust Graph Routing runtime for Apollo Federation
 
-![Version: 1.31.0-alpha.2](https://img.shields.io/badge/Version-1.31.0--alpha.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.2](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.2-informational?style=flat-square)
+![Version: 1.31.0-alpha.3](https://img.shields.io/badge/Version-1.31.0--alpha.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.3](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.3-informational?style=flat-square)
 
 ## Prerequisites
 
@@ -11,7 +11,7 @@
 ## Get Repo Info
 
 ```console
-helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.2
+helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.3
 ```
 
 ## Install Chart
@@ -19,7 +19,7 @@ helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.
 **Important:** only helm3 is supported
 
 ```console
-helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.2 --values my-values.yaml
+helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.3 --values my-values.yaml
 ```
 
 _See [configuration](#configuration) below._
diff --git a/scripts/install.sh b/scripts/install.sh
index 6d3c682c23..af0df5fdc8 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -11,7 +11,7 @@ BINARY_DOWNLOAD_PREFIX="https://github.com/apollographql/router/releases/downloa
 
 # Router version defined in apollo-router's Cargo.toml
 # Note: Change this line manually during the release steps.
-PACKAGE_VERSION="v1.31.0-alpha.2"
+PACKAGE_VERSION="v1.31.0-alpha.3"
 
 download_binary() {
     downloader --check

From 951789f4ad97c338de5709ef623472e4298a75b6 Mon Sep 17 00:00:00 2001
From: Geoffroy Couprie 
Date: Tue, 26 Sep 2023 14:19:28 +0200
Subject: [PATCH 48/51] remove broken migration (#3906)

This follows up on https://github.com/apollographql/router/pull/3852 but
removes the migration which was not practical to implement due to
limitations of our current migrator code. Unfortunately, this will mean
that users will need to transition from using the _experimental_ feature
(which is subject [our experimental release
stage](https://www.apollographql.com/docs/resources/product-launch-stages/))
to the GA release by updating their configuration accordingly.
---
 .../migrations/0011-experimental-http2.yaml   | 25 -------------------
 1 file changed, 25 deletions(-)
 delete mode 100644 apollo-router/src/configuration/migrations/0011-experimental-http2.yaml

diff --git a/apollo-router/src/configuration/migrations/0011-experimental-http2.yaml b/apollo-router/src/configuration/migrations/0011-experimental-http2.yaml
deleted file mode 100644
index e73b228a03..0000000000
--- a/apollo-router/src/configuration/migrations/0011-experimental-http2.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-description: Move experimental_enable_http2 to experimental_http2
-actions:
-  - type: change
-    path: traffic_shaping.all.experimental_enable_http2
-    from: true
-    to: enable
-  - type: change
-    path: traffic_shaping.all.experimental_enable_http2
-    from: false
-    to: disable
-  - type: move
-    from: traffic_shaping.all.experimental_enable_http2
-    to: traffic_shaping.all.experimental_http2
-
-  - type: change
-    path: traffic_shaping.subgraphs..experimental_enable_http2
-    from: true
-    to: enable
-  - type: change
-    path: traffic_shaping.subgraphs..experimental_enable_http2
-    from: false
-    to: disable
-  - type: move
-    from: traffic_shaping.subgraphs..experimental_enable_http2
-    to: traffic_shaping.subgraphs..experimental_http2

From 29066a5bdcd6cf15a153aac67ec17aa3dfbbc565 Mon Sep 17 00:00:00 2001
From: Jesse Rosenberger 
Date: Tue, 26 Sep 2023 13:05:52 +0000
Subject: [PATCH 49/51] prep release: v1.31.0-alpha.4

---
 Cargo.lock                                    |  6 ++--
 apollo-router-benchmarks/Cargo.toml           |  2 +-
 apollo-router-scaffold/Cargo.toml             |  2 +-
 .../templates/base/Cargo.toml                 |  2 +-
 .../templates/base/xtask/Cargo.toml           |  2 +-
 apollo-router/Cargo.toml                      |  2 +-
 .../tracing/docker-compose.datadog.yml        |  2 +-
 dockerfiles/tracing/docker-compose.jaeger.yml |  2 +-
 dockerfiles/tracing/docker-compose.zipkin.yml |  2 +-
 docs/source/containerization/docker.mdx       |  2 +-
 docs/source/containerization/kubernetes.mdx   | 28 +++++++++----------
 helm/chart/router/Chart.yaml                  |  4 +--
 helm/chart/router/README.md                   |  6 ++--
 scripts/install.sh                            |  2 +-
 14 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 96052fc58e..113b86a69d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -264,7 +264,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router"
-version = "1.31.0-alpha.3"
+version = "1.31.0-alpha.4"
 dependencies = [
  "access-json",
  "anyhow",
@@ -411,7 +411,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.3"
+version = "1.31.0-alpha.4"
 dependencies = [
  "apollo-parser 0.6.2",
  "apollo-router",
@@ -427,7 +427,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.3"
+version = "1.31.0-alpha.4"
 dependencies = [
  "anyhow",
  "cargo-scaffold",
diff --git a/apollo-router-benchmarks/Cargo.toml b/apollo-router-benchmarks/Cargo.toml
index f68de3e70e..65e513ba80 100644
--- a/apollo-router-benchmarks/Cargo.toml
+++ b/apollo-router-benchmarks/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.3"
+version = "1.31.0-alpha.4"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml
index 2e058aced7..afab3bfe16 100644
--- a/apollo-router-scaffold/Cargo.toml
+++ b/apollo-router-scaffold/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.3"
+version = "1.31.0-alpha.4"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/templates/base/Cargo.toml b/apollo-router-scaffold/templates/base/Cargo.toml
index b26035f57c..8e00a942bf 100644
--- a/apollo-router-scaffold/templates/base/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/Cargo.toml
@@ -22,7 +22,7 @@ apollo-router = { path ="{{integration_test}}apollo-router" }
 apollo-router = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
 # Note if you update these dependencies then also update xtask/Cargo.toml
-apollo-router = "1.31.0-alpha.3"
+apollo-router = "1.31.0-alpha.4"
 {{/if}}
 {{/if}}
 async-trait = "0.1.52"
diff --git a/apollo-router-scaffold/templates/base/xtask/Cargo.toml b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
index 376844c6ec..30bc08f644 100644
--- a/apollo-router-scaffold/templates/base/xtask/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
@@ -13,7 +13,7 @@ apollo-router-scaffold = { path ="{{integration_test}}apollo-router-scaffold" }
 {{#if branch}}
 apollo-router-scaffold = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
-apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.3" }
+apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.4" }
 {{/if}}
 {{/if}}
 anyhow = "1.0.58"
diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml
index df34444c55..b6dfc7ce39 100644
--- a/apollo-router/Cargo.toml
+++ b/apollo-router/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router"
-version = "1.31.0-alpha.3"
+version = "1.31.0-alpha.4"
 authors = ["Apollo Graph, Inc. "]
 repository = "https://github.com/apollographql/router/"
 documentation = "https://docs.rs/apollo-router"
diff --git a/dockerfiles/tracing/docker-compose.datadog.yml b/dockerfiles/tracing/docker-compose.datadog.yml
index f1a21c848e..5f12f785af 100644
--- a/dockerfiles/tracing/docker-compose.datadog.yml
+++ b/dockerfiles/tracing/docker-compose.datadog.yml
@@ -3,7 +3,7 @@ services:
 
   apollo-router:
     container_name: apollo-router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.3
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.4
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/datadog.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.jaeger.yml b/dockerfiles/tracing/docker-compose.jaeger.yml
index 25456fca33..19c83e15cd 100644
--- a/dockerfiles/tracing/docker-compose.jaeger.yml
+++ b/dockerfiles/tracing/docker-compose.jaeger.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     #build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.3
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.4
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/jaeger.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.zipkin.yml b/dockerfiles/tracing/docker-compose.zipkin.yml
index 857e30a98f..c6122bd9a7 100644
--- a/dockerfiles/tracing/docker-compose.zipkin.yml
+++ b/dockerfiles/tracing/docker-compose.zipkin.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.3
+    image: ghcr.io/apollographql/router:v1.31.0-alpha.4
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/zipkin.router.yaml:/etc/config/configuration.yaml
diff --git a/docs/source/containerization/docker.mdx b/docs/source/containerization/docker.mdx
index 31c10f9224..b0e93b2897 100644
--- a/docs/source/containerization/docker.mdx
+++ b/docs/source/containerization/docker.mdx
@@ -11,7 +11,7 @@ The default behaviour of the router images is suitable for a quickstart or devel
 
 Note: The [docker documentation](https://docs.docker.com/engine/reference/run/) for the run command may be helpful when reading through the examples.
 
-Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.3`
+Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.4`
 
 ## Override the configuration
 
diff --git a/docs/source/containerization/kubernetes.mdx b/docs/source/containerization/kubernetes.mdx
index 072b71cbc8..644eb6d9fa 100644
--- a/docs/source/containerization/kubernetes.mdx
+++ b/docs/source/containerization/kubernetes.mdx
@@ -13,7 +13,7 @@ import { Link } from 'gatsby';
 
 [Helm](https://helm.sh) is the package manager for kubernetes.
 
-There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.3/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
+There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.4/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
 
 In both the following examples, we are using helm to install the router:
  - into namespace "router-deploy" (create namespace if it doesn't exist)
@@ -64,10 +64,10 @@ kind: ServiceAccount
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.3
+    helm.sh/chart: router-1.31.0-alpha.4
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.3"
+    app.kubernetes.io/version: "v1.31.0-alpha.4"
     app.kubernetes.io/managed-by: Helm
 ---
 # Source: router/templates/secret.yaml
@@ -76,10 +76,10 @@ kind: Secret
 metadata:
   name: "release-name-router"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.3
+    helm.sh/chart: router-1.31.0-alpha.4
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.3"
+    app.kubernetes.io/version: "v1.31.0-alpha.4"
     app.kubernetes.io/managed-by: Helm
 data:
   managedFederationApiKey: "UkVEQUNURUQ="
@@ -90,10 +90,10 @@ kind: ConfigMap
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.3
+    helm.sh/chart: router-1.31.0-alpha.4
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.3"
+    app.kubernetes.io/version: "v1.31.0-alpha.4"
     app.kubernetes.io/managed-by: Helm
 data:
   configuration.yaml: |
@@ -117,10 +117,10 @@ kind: Service
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.3
+    helm.sh/chart: router-1.31.0-alpha.4
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.3"
+    app.kubernetes.io/version: "v1.31.0-alpha.4"
     app.kubernetes.io/managed-by: Helm
 spec:
   type: ClusterIP
@@ -143,10 +143,10 @@ kind: Deployment
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.3
+    helm.sh/chart: router-1.31.0-alpha.4
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.3"
+    app.kubernetes.io/version: "v1.31.0-alpha.4"
     app.kubernetes.io/managed-by: Helm
   
   annotations:
@@ -174,7 +174,7 @@ spec:
         - name: router
           securityContext:
             {}
-          image: "ghcr.io/apollographql/router:v1.31.0-alpha.3"
+          image: "ghcr.io/apollographql/router:v1.31.0-alpha.4"
           imagePullPolicy: IfNotPresent
           args:
             - --hot-reload
@@ -226,10 +226,10 @@ kind: Pod
 metadata:
   name: "release-name-router-test-connection"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.3
+    helm.sh/chart: router-1.31.0-alpha.4
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.3"
+    app.kubernetes.io/version: "v1.31.0-alpha.4"
     app.kubernetes.io/managed-by: Helm
   annotations:
     "helm.sh/hook": test
diff --git a/helm/chart/router/Chart.yaml b/helm/chart/router/Chart.yaml
index 14cb1a852d..1c70ef5a8a 100644
--- a/helm/chart/router/Chart.yaml
+++ b/helm/chart/router/Chart.yaml
@@ -20,10 +20,10 @@ type: application
 # so it matches the shape of our release process and release automation.
 # By proxy of that decision, this version uses SemVer 2.0.0, though the prefix
 # of "v" is not included.
-version: 1.31.0-alpha.3
+version: 1.31.0-alpha.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.31.0-alpha.3"
+appVersion: "v1.31.0-alpha.4"
diff --git a/helm/chart/router/README.md b/helm/chart/router/README.md
index eef24017d4..939ce9ec39 100644
--- a/helm/chart/router/README.md
+++ b/helm/chart/router/README.md
@@ -2,7 +2,7 @@
 
 [router](https://github.com/apollographql/router) Rust Graph Routing runtime for Apollo Federation
 
-![Version: 1.31.0-alpha.3](https://img.shields.io/badge/Version-1.31.0--alpha.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.3](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.3-informational?style=flat-square)
+![Version: 1.31.0-alpha.4](https://img.shields.io/badge/Version-1.31.0--alpha.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.4](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.4-informational?style=flat-square)
 
 ## Prerequisites
 
@@ -11,7 +11,7 @@
 ## Get Repo Info
 
 ```console
-helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.3
+helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.4
 ```
 
 ## Install Chart
@@ -19,7 +19,7 @@ helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.
 **Important:** only helm3 is supported
 
 ```console
-helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.3 --values my-values.yaml
+helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.4 --values my-values.yaml
 ```
 
 _See [configuration](#configuration) below._
diff --git a/scripts/install.sh b/scripts/install.sh
index af0df5fdc8..743f1ee55f 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -11,7 +11,7 @@ BINARY_DOWNLOAD_PREFIX="https://github.com/apollographql/router/releases/downloa
 
 # Router version defined in apollo-router's Cargo.toml
 # Note: Change this line manually during the release steps.
-PACKAGE_VERSION="v1.31.0-alpha.3"
+PACKAGE_VERSION="v1.31.0-alpha.4"
 
 download_binary() {
     downloader --check

From c602a0421a05a5971a4610690a13b238725fc69f Mon Sep 17 00:00:00 2001
From: Iryna Shestak 
Date: Wed, 27 Sep 2023 14:30:08 +0200
Subject: [PATCH 50/51] chore(changelog): 1.31.0 release changelog (#3910)

Co-authored-by: Jesse Rosenberger 
---
 .../docs_garypen_fix_rhai_subgraph_docs.md    |  2 +-
 .changesets/feat_bnjjj_fix_3621.md            |  8 ++--
 .changesets/feat_bnjjj_fix_3767.md            | 42 -------------------
 .../feat_garypen_one_shot_async_check.md      | 11 -----
 .changesets/feat_geal_h2c.md                  |  4 +-
 .../feat_geal_plan_cache_warmup_doc.md        | 36 ++++++++++++----
 .../feat_geal_reintroduce_rhai_json.md        |  2 +-
 .changesets/feat_geal_subgraph_mtls.md        |  2 +
 .../feat_geal_supergraph_coprocessor2.md      | 20 ++++-----
 .changesets/feat_sm_add_awsxray_propagator.md |  4 +-
 .../fix_bnjjj_fix_otel_data_from_log.md       |  5 ---
 .changesets/fix_garypen_3823_fix_diy.md       |  7 ----
 .changesets/fix_garypen_helm_extra_labels.md  |  2 +-
 ...gni_coprocessors_discard_content_length.md |  2 +-
 ...ted_queries_failover_and_error_handling.md |  4 +-
 .changesets/fix_renee_fix_large_ints.md       |  8 +---
 .changesets/fix_simon_large_id.md             |  2 +-
 .changesets/maint_bnjjj_fix_3740.md           |  5 ---
 .../maint_garypen_3663_no_overwrites.md       |  5 ---
 docs/source/configuration/overview.mdx        |  4 +-
 20 files changed, 60 insertions(+), 115 deletions(-)
 delete mode 100644 .changesets/feat_bnjjj_fix_3767.md
 delete mode 100644 .changesets/feat_garypen_one_shot_async_check.md
 delete mode 100644 .changesets/fix_bnjjj_fix_otel_data_from_log.md
 delete mode 100644 .changesets/fix_garypen_3823_fix_diy.md
 delete mode 100644 .changesets/maint_bnjjj_fix_3740.md
 delete mode 100644 .changesets/maint_garypen_3663_no_overwrites.md

diff --git a/.changesets/docs_garypen_fix_rhai_subgraph_docs.md b/.changesets/docs_garypen_fix_rhai_subgraph_docs.md
index 94f696f5fa..d08bf3f967 100644
--- a/.changesets/docs_garypen_fix_rhai_subgraph_docs.md
+++ b/.changesets/docs_garypen_fix_rhai_subgraph_docs.md
@@ -1,5 +1,5 @@
 ### Rhai documentation: remove incorrect statement about request.subgraph fields ([PR #3808](https://github.com/apollographql/router/pull/3808))
 
-It is possible to modify `request.subgraph` fields from a `Rhai` script and so the documentation should reflect that.
+It is possible to modify `request.subgraph` fields from a Rhai script, which is now correctly reflected in [Rhai documentation](https://www.apollographql.com/docs/router/customizations/rhai-api/#request-interface).
 
 By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3808
\ No newline at end of file
diff --git a/.changesets/feat_bnjjj_fix_3621.md b/.changesets/feat_bnjjj_fix_3621.md
index c09c2619a5..dacac24f7d 100644
--- a/.changesets/feat_bnjjj_fix_3621.md
+++ b/.changesets/feat_bnjjj_fix_3621.md
@@ -1,14 +1,14 @@
-### feat(redis): add configuration to set the timeout ([Issue #3621](https://github.com/apollographql/router/issues/3621))
+### Added configuration to set redis request timeout ([Issue #3621](https://github.com/apollographql/router/issues/3621))
 
-It adds a configuration to set another timeout than the default one (2ms) for redis requests. It also change the default timeout to 2ms (previously set to 1ms)
+We added configuration to override default timeout for Redis requests. Default timeout was also changed from 1ms to **2ms**.
 
-Example for APQ:
+Here is an example to change the timeout for [Distributed APQ](https://www.apollographql.com/docs/router/configuration/distributed-caching#distributed-apq-caching) (an Enterprise Feature):
 ```yaml
 apq:
   router:
     cache:
       redis:
-        urls: ["redis://..."] 
+        urls: ["redis://..."]
         timeout: 5ms
 ```
 
diff --git a/.changesets/feat_bnjjj_fix_3767.md b/.changesets/feat_bnjjj_fix_3767.md
deleted file mode 100644
index a79b1e059d..0000000000
--- a/.changesets/feat_bnjjj_fix_3767.md
+++ /dev/null
@@ -1,42 +0,0 @@
-### feat(telemetry): add metrics for query plan warmup and schema load ([Issue #3767](https://github.com/apollographql/router/issues/3767))
-
-It adds histogram metrics for `apollo_router_query_planning_warmup_duration` and `apollo_router_schema_load_duration`.
-
-Example in Prometheus:
-
-```
-# HELP apollo_router_query_planning_warmup_duration apollo_router_query_planning_warmup_duration
-# TYPE apollo_router_query_planning_warmup_duration histogram
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.5"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="2.5"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="5"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="10"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="20"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="100"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1000"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="+Inf"} 1
-apollo_router_query_planning_warmup_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.022390619
-apollo_router_query_planning_warmup_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 1
-# HELP apollo_router_schema_load_duration apollo_router_schema_load_duration
-# TYPE apollo_router_schema_load_duration histogram
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.5"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="2.5"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="5"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="10"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="20"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="100"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="1000"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="+Inf"} 8
-apollo_router_schema_load_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.023486205999999996
-apollo_router_schema_load_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 8
-```
-
-By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3807
diff --git a/.changesets/feat_garypen_one_shot_async_check.md b/.changesets/feat_garypen_one_shot_async_check.md
deleted file mode 100644
index 61298d848c..0000000000
--- a/.changesets/feat_garypen_one_shot_async_check.md
+++ /dev/null
@@ -1,11 +0,0 @@
-### Introduce OneShotAsyncCheckpoint ([PR #3819](https://github.com/apollographql/router/pull/3819))
-
-The existing AsynCheckpoint requires `Clone` and thus introduces a need for Service Buffering which reduces the performance and resiliance of the router.
-
-This new set of Services, Layers and utility functions removes the requirement for `Clone` and thus the requirement for service buffering.
-
-Existing uses of AsyncCheckpoint within the router are replaced with OneShotAsyncCheckpoint along with the requirement to `buffer()` such services.
-
-If you have a custom plugin that makes use of `AsyncCheckpoint`, we encourage you to migrate to `OneShotAsyncCheckpoint` and thus reduce the requirement for service buffering from your router.
-
-By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3819
\ No newline at end of file
diff --git a/.changesets/feat_geal_h2c.md b/.changesets/feat_geal_h2c.md
index 0852c40671..b6874069a0 100644
--- a/.changesets/feat_geal_h2c.md
+++ b/.changesets/feat_geal_h2c.md
@@ -1,5 +1,5 @@
-### HTTP/2 Cleartext protocol (h2c) support for subgraph connections ([Issue #3535](https://github.com/apollographql/router/issues/3535))
+### HTTP/2 Cleartext protocol (H2C) support for subgraph connections ([Issue #3535](https://github.com/apollographql/router/issues/3535))
 
-The router can now connect to subgraphs over HTTP/2 Cleartext, which uses the HTTP/2 binary protocol directly over TCP without TLS. To activate it, set the `experimental_http2` option to `http2_only`.
+The router can now connect to subgraphs over HTTP/2 Cleartext (H2C), which uses the HTTP/2 binary protocol directly over TCP **without TLS**, which is a mode of operation desired with some service mesh configurations (e.g., Istio, Envoy) where the value of added encryption is unnecessary. To activate it, set the `experimental_http2` option to `http2_only`.
 
 By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3852
\ No newline at end of file
diff --git a/.changesets/feat_geal_plan_cache_warmup_doc.md b/.changesets/feat_geal_plan_cache_warmup_doc.md
index a7c9a6f2f1..8eb3e1dee1 100644
--- a/.changesets/feat_geal_plan_cache_warmup_doc.md
+++ b/.changesets/feat_geal_plan_cache_warmup_doc.md
@@ -1,13 +1,35 @@
-### Query plan cache warm-up improvements ([Issue #3704](https://github.com/apollographql/router/issues/3704))
+### Query plan cache warm-up improvements ([Issue #3704](https://github.com/apollographql/router/issues/3704), [Issue #3767](https://github.com/apollographql/router/issues/3767))
 
-The `warm_up_queries` option enables quicker schema updates by precomputing query plans for your most used cached queries and your persisted queries. When a new schema is loaded, a precomputed query plan for it may already be in in-memory cache.
+The `warm_up_queries` option enables quicker schema updates by precomputing query plans for your most used cached queries and your persisted queries. When a new schema is loaded, a precomputed query plan for it may already be in the in-memory cache.
 
-We made a series of improvements to this feature to make it more usable:
-* It is now active by default and warms up the cache with the 30% most used queries of the previous cache. The amount is still configurable, and it can be deactivated by setting it to 0.
-* We added new metrics to track the time spent loading a new schema and planning queries in the warm-up phase. You can also measure the query plan cache usage, to know how many entries are used, and the cache hit rate, both for the in memory cache and the distributed cache.
-* The warm-up will now plan queries in random order, to make sure that the work can be shared by multiple router instances using distributed caching
+We made a series of improvements to this feature to make it easier to use:
+* It is now active by default.
+* It warms up the cache with the 30% most used queries from previous cache.
+* The query cache percentage continues to be configurable, and it can be deactivated by setting it to 0.
+* The warm-up will now plan queries in random order to make sure that the work can be shared by multiple router instances using distributed caching.
 * Persisted queries are part of the warmed up queries.
 
+We also added histogram metrics for `apollo_router_query_planning_warmup_duration` and `apollo_router_schema_load_duration`. These metrics make it easier to track the time spent loading a new schema and planning queries in the warm-up phase. You can measure the query plan cache usage for both the in-memory-cache and distributed cache. This makes it easier to know how many entries are used as well as the cache hit rate.
+
+Here is what these metrics would look like in Prometheus:
+
+```
+# HELP apollo_router_query_planning_warmup_duration apollo_router_query_planning_warmup_duration
+# TYPE apollo_router_query_planning_warmup_duration histogram
+apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 1
+apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 1
+apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 1
+apollo_router_query_planning_warmup_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.022390619
+apollo_router_query_planning_warmup_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 1
+# HELP apollo_router_schema_load_duration apollo_router_schema_load_duration
+# TYPE apollo_router_schema_load_duration histogram
+apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 8
+apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 8
+apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 8
+```
+
 You can get more information about operating the query plan cache and its warm-up phase in the [documentation](https://www.apollographql.com/docs/router/configuration/in-memory-caching#cache-warm-up)
 
-By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3815 https://github.com/apollographql/router/pull/3801 https://github.com/apollographql/router/pull/3767 https://github.com/apollographql/router/pull/3769 https://github.com/apollographql/router/pull/3770
\ No newline at end of file
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3815 https://github.com/apollographql/router/pull/3801 https://github.com/apollographql/router/pull/3767 https://github.com/apollographql/router/pull/3769 https://github.com/apollographql/router/pull/3770
+
+By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3807
\ No newline at end of file
diff --git a/.changesets/feat_geal_reintroduce_rhai_json.md b/.changesets/feat_geal_reintroduce_rhai_json.md
index f76e238eca..7f960707db 100644
--- a/.changesets/feat_geal_reintroduce_rhai_json.md
+++ b/.changesets/feat_geal_reintroduce_rhai_json.md
@@ -1,5 +1,5 @@
 ### JSON encoding and decoding in Rhai ([PR #3785](https://github.com/apollographql/router/pull/3785))
 
-It is now possible to encode or decode JSON from Rhai scripts using `json::encode` and `json::decode`
+It is now possible to encode or decode JSON from Rhai scripts using `json::encode` and `json::decode`.
 
 By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3785
\ No newline at end of file
diff --git a/.changesets/feat_geal_subgraph_mtls.md b/.changesets/feat_geal_subgraph_mtls.md
index a377ab52bf..9ab4698a1a 100644
--- a/.changesets/feat_geal_subgraph_mtls.md
+++ b/.changesets/feat_geal_subgraph_mtls.md
@@ -18,4 +18,6 @@ tls:
           key: ${file./path/to/key.pem}
 ```
 
+Details on TLS client authentication can be found in the [documentation](https://www.apollographql.com/docs/router/configuration/overview#tls-client-authentication-for-subgraph-requests)
+
 By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3794
\ No newline at end of file
diff --git a/.changesets/feat_geal_supergraph_coprocessor2.md b/.changesets/feat_geal_supergraph_coprocessor2.md
index 9dd3030a34..ead15c859c 100644
--- a/.changesets/feat_geal_supergraph_coprocessor2.md
+++ b/.changesets/feat_geal_supergraph_coprocessor2.md
@@ -1,6 +1,6 @@
 ### Supergraph coprocessor implementation ([PR #3647](https://github.com/apollographql/router/pull/3647))
 
-Coprocessors now support supergraph service interception. 
+Coprocessors now support supergraph service interception.
 
 On the request side, the coprocessor payload can contain:
 - method
@@ -17,17 +17,17 @@ On the response side, the payload can contain:
 - sdl
 
 The supergraph request body contains:
-* query
-* operation name
-* variables
-* extensions
+- query
+- operation name
+- variables
+- extensions
 
 The supergraph response body contains:
-* label
-* data
-* errors
-* extensions
+- label
+- data
+- errors
+- extensions
 
-When using `@defer` or subscriptions a supergraph response may contain multiple GraphQL responses, and the coprocessor will be called for each.
+When using `@defer` or subscriptions a supergraph response may contain multiple GraphQL responses. The coprocessor will be called for each response. Please refer to our [coprocessor documentation](https://www.apollographql.com/docs/router/customizations/coprocessor) for more information.
 
 By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3647
\ No newline at end of file
diff --git a/.changesets/feat_sm_add_awsxray_propagator.md b/.changesets/feat_sm_add_awsxray_propagator.md
index 9b1ff8b2e7..fbe21cd136 100644
--- a/.changesets/feat_sm_add_awsxray_propagator.md
+++ b/.changesets/feat_sm_add_awsxray_propagator.md
@@ -1,5 +1,5 @@
-Adds support for the OpenTelemetry AWS X-Ray tracing propagator.
+### Adds support for the OpenTelemetry AWS X-Ray tracing propagator ([PR #3580](https://github.com/apollographql/router/pull/3580))
 
-This propagator helps propagate tracing information from upstream services (such as AWS load balancers) to downstream services and handles conversion between the X-Ray trace id format and OpenTelemetry span contexts.
+This propagator helps propagate tracing information from upstream services (such as AWS load balancers) to downstream services. It also handles conversion between the X-Ray trace id format and OpenTelemetry span contexts.
 
 By [@scottmace](https://github.com/scottmace) in https://github.com/apollographql/router/pull/3580
diff --git a/.changesets/fix_bnjjj_fix_otel_data_from_log.md b/.changesets/fix_bnjjj_fix_otel_data_from_log.md
deleted file mode 100644
index 8d2fb003cc..0000000000
--- a/.changesets/fix_bnjjj_fix_otel_data_from_log.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### fix(telemetry): do not display otel bug if the trace is sampled ([PR #3832](https://github.com/apollographql/router/pull/3832))
-
-We changed the way we are sampling spans. If you had logs in an evicted span it displays that log `Unable to find OtelData in extensions; this is a bug` which is incorrect, it's not a bug, we just can't display the `trace_id` because there is no `trace_id` as the span has not been sampled.
-
-By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3832
\ No newline at end of file
diff --git a/.changesets/fix_garypen_3823_fix_diy.md b/.changesets/fix_garypen_3823_fix_diy.md
deleted file mode 100644
index 3da4807001..0000000000
--- a/.changesets/fix_garypen_3823_fix_diy.md
+++ /dev/null
@@ -1,7 +0,0 @@
-### fix the diy build_docker_image.sh script ([Issue #3823](https://github.com/apollographql/router/issues/3823))
-
-The diy `build_docker_image.sh` script was broken by the latest `deno` update due to a new requirement to have `cmake` when compiling the router.
-
-This adds `cmake` to the build step for diy builds to fix the problem.
-
-By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3824
diff --git a/.changesets/fix_garypen_helm_extra_labels.md b/.changesets/fix_garypen_helm_extra_labels.md
index 4082915ff1..d8439dad01 100644
--- a/.changesets/fix_garypen_helm_extra_labels.md
+++ b/.changesets/fix_garypen_helm_extra_labels.md
@@ -1,7 +1,7 @@
 ### Helm: If there are `extraLabels` add them to all resources ([PR #3622](https://github.com/apollographql/router/pull/3622))
 
 This extends the functionality of `extraLabels` so that, if they are defined, they will be templated for all resources created by the chart.
-    
+
 Previously, they were only templated onto the `Deployment` resource.
 
 By [@garypen](https://github.com/garypen) and [@bjoernw](https://github.com/bjoernw) in https://github.com/apollographql/router/pull/3622
diff --git a/.changesets/fix_igni_coprocessors_discard_content_length.md b/.changesets/fix_igni_coprocessors_discard_content_length.md
index 3655239e87..16c02d3494 100644
--- a/.changesets/fix_igni_coprocessors_discard_content_length.md
+++ b/.changesets/fix_igni_coprocessors_discard_content_length.md
@@ -1,4 +1,4 @@
-### Coprocessors: Discard content-length sent by coprocessors. ([PR #3802](https://github.com/apollographql/router/pull/3802))
+### Coprocessors: Discard content-length sent by coprocessors ([PR #3802](https://github.com/apollographql/router/pull/3802))
 
 The `content-length` of an HTTP response can only be computed when a router response is being sent.
 We now discard coprocessors `content-length` header to make sure the value is computed correctly.
diff --git a/.changesets/fix_persisted_queries_failover_and_error_handling.md b/.changesets/fix_persisted_queries_failover_and_error_handling.md
index 554ec346f0..26ed4644ad 100644
--- a/.changesets/fix_persisted_queries_failover_and_error_handling.md
+++ b/.changesets/fix_persisted_queries_failover_and_error_handling.md
@@ -1,5 +1,5 @@
-### Improve multi-cloud failover and error handling for Persisted Queries
+### Improve multi-cloud failover and error handling for Persisted Queries ([PR #3863](https://github.com/apollographql/router/pull/3863))
 
-Improves the resilience of the Persisted Queries feature to Uplink outages, and makes errors fetching Persisted Query Manifests from Uplink more visible.
+Improves the resilience of the Persisted Queries feature to Uplink outages. This makes errors while fetching persisted query manifests from Uplink more visible.
 
 By [@glasser](https://github.com/glasser) in https://github.com/apollographql/router/pull/3863
\ No newline at end of file
diff --git a/.changesets/fix_renee_fix_large_ints.md b/.changesets/fix_renee_fix_large_ints.md
index 756249f92c..1ab697866d 100644
--- a/.changesets/fix_renee_fix_large_ints.md
+++ b/.changesets/fix_renee_fix_large_ints.md
@@ -1,4 +1,4 @@
-### Fix error response on large numbers in query transformations ([PR #3820](https://github.com/apollographql/router/pull/3820))
+### Fix error response on large number types in query transformations ([PR #3820](https://github.com/apollographql/router/pull/3820))
 
 This bug caused the router to reject operations where a large hardcoded integer was used as input for a Float field:
 
@@ -13,10 +13,6 @@ type Query {
 }
 ```
 
-Now the number is correctly interpreted as a Float.
-This bug only affected hardcoded numbers, not numbers provided through variables.
-
-
----
+This number is now correctly interpreted as a `Float`. This bug only affected hardcoded numbers, not numbers provided through variables.
 
 By [@goto-bus-stop](https://github.com/goto-bus-stop) in https://github.com/apollographql/router/pull/3820
\ No newline at end of file
diff --git a/.changesets/fix_simon_large_id.md b/.changesets/fix_simon_large_id.md
index 58fdfcdebe..d2d0a28733 100644
--- a/.changesets/fix_simon_large_id.md
+++ b/.changesets/fix_simon_large_id.md
@@ -1,4 +1,4 @@
-### Fix validation error with ID variable values overflowing i32 ([Issue #3873](https://github.com/apollographql/router/issues/3873))
+### Fix validation error with `ID` variable values overflowing 32-bit integers ([Issue #3873](https://github.com/apollographql/router/issues/3873))
 
 Input values for variables of type `ID` were previously validated as "either like a GraphQL `Int` or like a GraphQL `String`". GraphQL `Int` is specified as a signed 32-bit integer, such that values that overflow fail validation. Applying this range restriction to `ID` values was incorrect. Instead, validation for `ID` now accepts any JSON integer or JSON string value, so that IDs larger than 32 bits can be used.
 
diff --git a/.changesets/maint_bnjjj_fix_3740.md b/.changesets/maint_bnjjj_fix_3740.md
deleted file mode 100644
index d4bc5fee7d..0000000000
--- a/.changesets/maint_bnjjj_fix_3740.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### Add integration test for subscription ([Issue #3740](https://github.com/apollographql/router/issues/3740))
-
-Add a regression test that ensures an unnamed subscription makes it through the request pipeline.
-
-By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3752
\ No newline at end of file
diff --git a/.changesets/maint_garypen_3663_no_overwrites.md b/.changesets/maint_garypen_3663_no_overwrites.md
deleted file mode 100644
index 1a155f7353..0000000000
--- a/.changesets/maint_garypen_3663_no_overwrites.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### Check if docker images and helm charts exist before creating them ([Issue #3663](https://github.com/apollographql/router/issues/3663))
-
-This improves the resilience of our pipeline.
-
-By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3825
\ No newline at end of file
diff --git a/docs/source/configuration/overview.mdx b/docs/source/configuration/overview.mdx
index b96f4fcc2a..4b2125290d 100644
--- a/docs/source/configuration/overview.mdx
+++ b/docs/source/configuration/overview.mdx
@@ -238,7 +238,7 @@ If set, disables sending anonymous usage information to Apollo.
 
 
 
-If set, the listen address of the router. 
+If set, the listen address of the router.
 
 
 
@@ -607,7 +607,7 @@ This produces the file as `server.crt` which can be used in `certificate_authori
 
 #### TLS client authentication for subgraph requests
 
-The router support mutual TLS authentication (mTLS) with the subgraphs. This means that it can authenticate itself to the subgraph using a certificate chain and a cryptographic key. It can be configured as follows:
+The router supports mutual TLS authentication (mTLS) with the subgraphs. This means that it can authenticate itself to the subgraph using a certificate chain and a cryptographic key. It can be configured as follows:
 
 ```yaml
 tls:

From f577d834a7e227662a05d2679e76a1f8ddfb32f3 Mon Sep 17 00:00:00 2001
From: Iryna Shestak 
Date: Wed, 27 Sep 2023 15:48:44 +0200
Subject: [PATCH 51/51] prep release: v1.31.0

---
 .../docs_garypen_fix_rhai_subgraph_docs.md    |   5 -
 .changesets/feat_bnjjj_fix_3621.md            |  15 --
 .changesets/feat_geal_h2c.md                  |   5 -
 .../feat_geal_plan_cache_warmup_doc.md        |  35 ----
 .../feat_geal_reintroduce_rhai_json.md        |   5 -
 .changesets/feat_geal_subgraph_mtls.md        |  23 ---
 .../feat_geal_supergraph_coprocessor2.md      |  33 ---
 .changesets/feat_sm_add_awsxray_propagator.md |   5 -
 .changesets/fix_garypen_helm_extra_labels.md  |   7 -
 ...gni_coprocessors_discard_content_length.md |   6 -
 ...ted_queries_failover_and_error_handling.md |   5 -
 .changesets/fix_renee_fix_large_ints.md       |  18 --
 .changesets/fix_simon_large_id.md             |   5 -
 CHANGELOG.md                                  | 190 ++++++++++++++++++
 Cargo.lock                                    |   6 +-
 apollo-router-benchmarks/Cargo.toml           |   2 +-
 apollo-router-scaffold/Cargo.toml             |   2 +-
 .../templates/base/Cargo.toml                 |   2 +-
 .../templates/base/xtask/Cargo.toml           |   2 +-
 apollo-router/Cargo.toml                      |   2 +-
 .../tracing/docker-compose.datadog.yml        |   2 +-
 dockerfiles/tracing/docker-compose.jaeger.yml |   2 +-
 dockerfiles/tracing/docker-compose.zipkin.yml |   2 +-
 docs/source/containerization/docker.mdx       |   2 +-
 docs/source/containerization/kubernetes.mdx   |  28 +--
 docs/source/federation-version-support.mdx    |  10 +-
 helm/chart/router/Chart.yaml                  |   4 +-
 helm/chart/router/README.md                   |   8 +-
 licenses.html                                 |   6 +-
 scripts/install.sh                            |   2 +-
 30 files changed, 234 insertions(+), 205 deletions(-)
 delete mode 100644 .changesets/docs_garypen_fix_rhai_subgraph_docs.md
 delete mode 100644 .changesets/feat_bnjjj_fix_3621.md
 delete mode 100644 .changesets/feat_geal_h2c.md
 delete mode 100644 .changesets/feat_geal_plan_cache_warmup_doc.md
 delete mode 100644 .changesets/feat_geal_reintroduce_rhai_json.md
 delete mode 100644 .changesets/feat_geal_subgraph_mtls.md
 delete mode 100644 .changesets/feat_geal_supergraph_coprocessor2.md
 delete mode 100644 .changesets/feat_sm_add_awsxray_propagator.md
 delete mode 100644 .changesets/fix_garypen_helm_extra_labels.md
 delete mode 100644 .changesets/fix_igni_coprocessors_discard_content_length.md
 delete mode 100644 .changesets/fix_persisted_queries_failover_and_error_handling.md
 delete mode 100644 .changesets/fix_renee_fix_large_ints.md
 delete mode 100644 .changesets/fix_simon_large_id.md

diff --git a/.changesets/docs_garypen_fix_rhai_subgraph_docs.md b/.changesets/docs_garypen_fix_rhai_subgraph_docs.md
deleted file mode 100644
index d08bf3f967..0000000000
--- a/.changesets/docs_garypen_fix_rhai_subgraph_docs.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### Rhai documentation: remove incorrect statement about request.subgraph fields ([PR #3808](https://github.com/apollographql/router/pull/3808))
-
-It is possible to modify `request.subgraph` fields from a Rhai script, which is now correctly reflected in [Rhai documentation](https://www.apollographql.com/docs/router/customizations/rhai-api/#request-interface).
-
-By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3808
\ No newline at end of file
diff --git a/.changesets/feat_bnjjj_fix_3621.md b/.changesets/feat_bnjjj_fix_3621.md
deleted file mode 100644
index dacac24f7d..0000000000
--- a/.changesets/feat_bnjjj_fix_3621.md
+++ /dev/null
@@ -1,15 +0,0 @@
-### Added configuration to set redis request timeout ([Issue #3621](https://github.com/apollographql/router/issues/3621))
-
-We added configuration to override default timeout for Redis requests. Default timeout was also changed from 1ms to **2ms**.
-
-Here is an example to change the timeout for [Distributed APQ](https://www.apollographql.com/docs/router/configuration/distributed-caching#distributed-apq-caching) (an Enterprise Feature):
-```yaml
-apq:
-  router:
-    cache:
-      redis:
-        urls: ["redis://..."]
-        timeout: 5ms
-```
-
-By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3817
\ No newline at end of file
diff --git a/.changesets/feat_geal_h2c.md b/.changesets/feat_geal_h2c.md
deleted file mode 100644
index b6874069a0..0000000000
--- a/.changesets/feat_geal_h2c.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### HTTP/2 Cleartext protocol (H2C) support for subgraph connections ([Issue #3535](https://github.com/apollographql/router/issues/3535))
-
-The router can now connect to subgraphs over HTTP/2 Cleartext (H2C), which uses the HTTP/2 binary protocol directly over TCP **without TLS**, which is a mode of operation desired with some service mesh configurations (e.g., Istio, Envoy) where the value of added encryption is unnecessary. To activate it, set the `experimental_http2` option to `http2_only`.
-
-By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3852
\ No newline at end of file
diff --git a/.changesets/feat_geal_plan_cache_warmup_doc.md b/.changesets/feat_geal_plan_cache_warmup_doc.md
deleted file mode 100644
index 8eb3e1dee1..0000000000
--- a/.changesets/feat_geal_plan_cache_warmup_doc.md
+++ /dev/null
@@ -1,35 +0,0 @@
-### Query plan cache warm-up improvements ([Issue #3704](https://github.com/apollographql/router/issues/3704), [Issue #3767](https://github.com/apollographql/router/issues/3767))
-
-The `warm_up_queries` option enables quicker schema updates by precomputing query plans for your most used cached queries and your persisted queries. When a new schema is loaded, a precomputed query plan for it may already be in the in-memory cache.
-
-We made a series of improvements to this feature to make it easier to use:
-* It is now active by default.
-* It warms up the cache with the 30% most used queries from previous cache.
-* The query cache percentage continues to be configurable, and it can be deactivated by setting it to 0.
-* The warm-up will now plan queries in random order to make sure that the work can be shared by multiple router instances using distributed caching.
-* Persisted queries are part of the warmed up queries.
-
-We also added histogram metrics for `apollo_router_query_planning_warmup_duration` and `apollo_router_schema_load_duration`. These metrics make it easier to track the time spent loading a new schema and planning queries in the warm-up phase. You can measure the query plan cache usage for both the in-memory-cache and distributed cache. This makes it easier to know how many entries are used as well as the cache hit rate.
-
-Here is what these metrics would look like in Prometheus:
-
-```
-# HELP apollo_router_query_planning_warmup_duration apollo_router_query_planning_warmup_duration
-# TYPE apollo_router_query_planning_warmup_duration histogram
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 1
-apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 1
-apollo_router_query_planning_warmup_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.022390619
-apollo_router_query_planning_warmup_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 1
-# HELP apollo_router_schema_load_duration apollo_router_schema_load_duration
-# TYPE apollo_router_schema_load_duration histogram
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 8
-apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 8
-```
-
-You can get more information about operating the query plan cache and its warm-up phase in the [documentation](https://www.apollographql.com/docs/router/configuration/in-memory-caching#cache-warm-up)
-
-By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3815 https://github.com/apollographql/router/pull/3801 https://github.com/apollographql/router/pull/3767 https://github.com/apollographql/router/pull/3769 https://github.com/apollographql/router/pull/3770
-
-By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3807
\ No newline at end of file
diff --git a/.changesets/feat_geal_reintroduce_rhai_json.md b/.changesets/feat_geal_reintroduce_rhai_json.md
deleted file mode 100644
index 7f960707db..0000000000
--- a/.changesets/feat_geal_reintroduce_rhai_json.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### JSON encoding and decoding in Rhai ([PR #3785](https://github.com/apollographql/router/pull/3785))
-
-It is now possible to encode or decode JSON from Rhai scripts using `json::encode` and `json::decode`.
-
-By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3785
\ No newline at end of file
diff --git a/.changesets/feat_geal_subgraph_mtls.md b/.changesets/feat_geal_subgraph_mtls.md
deleted file mode 100644
index 9ab4698a1a..0000000000
--- a/.changesets/feat_geal_subgraph_mtls.md
+++ /dev/null
@@ -1,23 +0,0 @@
-### TLS client authentication for subgraph requests ([Issue #3414](https://github.com/apollographql/router/issues/3414))
-
-The router now supports TLS client authentication when connecting to subgraphs. It can be configured as follows:
-
-```yaml
-tls:
-  subgraph:
-    all:
-      client_authentication:
-        certificate_chain: ${file./path/to/certificate_chain.pem}
-        key: ${file./path/to/key.pem}
-    # if configuring for a specific subgraph:
-    subgraphs:
-      # subgraph name
-      products:
-        client_authentication:
-          certificate_chain: ${file./path/to/certificate_chain.pem}
-          key: ${file./path/to/key.pem}
-```
-
-Details on TLS client authentication can be found in the [documentation](https://www.apollographql.com/docs/router/configuration/overview#tls-client-authentication-for-subgraph-requests)
-
-By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3794
\ No newline at end of file
diff --git a/.changesets/feat_geal_supergraph_coprocessor2.md b/.changesets/feat_geal_supergraph_coprocessor2.md
deleted file mode 100644
index ead15c859c..0000000000
--- a/.changesets/feat_geal_supergraph_coprocessor2.md
+++ /dev/null
@@ -1,33 +0,0 @@
-### Supergraph coprocessor implementation ([PR #3647](https://github.com/apollographql/router/pull/3647))
-
-Coprocessors now support supergraph service interception.
-
-On the request side, the coprocessor payload can contain:
-- method
-- headers
-- body
-- context
-- sdl
-
-On the response side, the payload can contain:
-- status_code
-- headers
-- body
-- context
-- sdl
-
-The supergraph request body contains:
-- query
-- operation name
-- variables
-- extensions
-
-The supergraph response body contains:
-- label
-- data
-- errors
-- extensions
-
-When using `@defer` or subscriptions a supergraph response may contain multiple GraphQL responses. The coprocessor will be called for each response. Please refer to our [coprocessor documentation](https://www.apollographql.com/docs/router/customizations/coprocessor) for more information.
-
-By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3647
\ No newline at end of file
diff --git a/.changesets/feat_sm_add_awsxray_propagator.md b/.changesets/feat_sm_add_awsxray_propagator.md
deleted file mode 100644
index fbe21cd136..0000000000
--- a/.changesets/feat_sm_add_awsxray_propagator.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### Adds support for the OpenTelemetry AWS X-Ray tracing propagator ([PR #3580](https://github.com/apollographql/router/pull/3580))
-
-This propagator helps propagate tracing information from upstream services (such as AWS load balancers) to downstream services. It also handles conversion between the X-Ray trace id format and OpenTelemetry span contexts.
-
-By [@scottmace](https://github.com/scottmace) in https://github.com/apollographql/router/pull/3580
diff --git a/.changesets/fix_garypen_helm_extra_labels.md b/.changesets/fix_garypen_helm_extra_labels.md
deleted file mode 100644
index d8439dad01..0000000000
--- a/.changesets/fix_garypen_helm_extra_labels.md
+++ /dev/null
@@ -1,7 +0,0 @@
-### Helm: If there are `extraLabels` add them to all resources ([PR #3622](https://github.com/apollographql/router/pull/3622))
-
-This extends the functionality of `extraLabels` so that, if they are defined, they will be templated for all resources created by the chart.
-
-Previously, they were only templated onto the `Deployment` resource.
-
-By [@garypen](https://github.com/garypen) and [@bjoernw](https://github.com/bjoernw) in https://github.com/apollographql/router/pull/3622
diff --git a/.changesets/fix_igni_coprocessors_discard_content_length.md b/.changesets/fix_igni_coprocessors_discard_content_length.md
deleted file mode 100644
index 16c02d3494..0000000000
--- a/.changesets/fix_igni_coprocessors_discard_content_length.md
+++ /dev/null
@@ -1,6 +0,0 @@
-### Coprocessors: Discard content-length sent by coprocessors ([PR #3802](https://github.com/apollographql/router/pull/3802))
-
-The `content-length` of an HTTP response can only be computed when a router response is being sent.
-We now discard coprocessors `content-length` header to make sure the value is computed correctly.
-
-By [@o0Ignition0o](https://github.com/o0Ignition0o) in https://github.com/apollographql/router/pull/3802
diff --git a/.changesets/fix_persisted_queries_failover_and_error_handling.md b/.changesets/fix_persisted_queries_failover_and_error_handling.md
deleted file mode 100644
index 26ed4644ad..0000000000
--- a/.changesets/fix_persisted_queries_failover_and_error_handling.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### Improve multi-cloud failover and error handling for Persisted Queries ([PR #3863](https://github.com/apollographql/router/pull/3863))
-
-Improves the resilience of the Persisted Queries feature to Uplink outages. This makes errors while fetching persisted query manifests from Uplink more visible.
-
-By [@glasser](https://github.com/glasser) in https://github.com/apollographql/router/pull/3863
\ No newline at end of file
diff --git a/.changesets/fix_renee_fix_large_ints.md b/.changesets/fix_renee_fix_large_ints.md
deleted file mode 100644
index 1ab697866d..0000000000
--- a/.changesets/fix_renee_fix_large_ints.md
+++ /dev/null
@@ -1,18 +0,0 @@
-### Fix error response on large number types in query transformations ([PR #3820](https://github.com/apollographql/router/pull/3820))
-
-This bug caused the router to reject operations where a large hardcoded integer was used as input for a Float field:
-
-```graphql
-# Schema
-type Query {
-    field(argument: Float): Int!
-}
-# Operation
-{
-    field(argument: 123456789123)
-}
-```
-
-This number is now correctly interpreted as a `Float`. This bug only affected hardcoded numbers, not numbers provided through variables.
-
-By [@goto-bus-stop](https://github.com/goto-bus-stop) in https://github.com/apollographql/router/pull/3820
\ No newline at end of file
diff --git a/.changesets/fix_simon_large_id.md b/.changesets/fix_simon_large_id.md
deleted file mode 100644
index d2d0a28733..0000000000
--- a/.changesets/fix_simon_large_id.md
+++ /dev/null
@@ -1,5 +0,0 @@
-### Fix validation error with `ID` variable values overflowing 32-bit integers ([Issue #3873](https://github.com/apollographql/router/issues/3873))
-
-Input values for variables of type `ID` were previously validated as "either like a GraphQL `Int` or like a GraphQL `String`". GraphQL `Int` is specified as a signed 32-bit integer, such that values that overflow fail validation. Applying this range restriction to `ID` values was incorrect. Instead, validation for `ID` now accepts any JSON integer or JSON string value, so that IDs larger than 32 bits can be used.
-
-By [@SimonSapin](https://github.com/SimonSapin) in https://github.com/apollographql/router/pull/3896
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ea9deb23a7..dfacf73f40 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,196 @@ All notable changes to Router will be documented in this file.
 
 This project adheres to [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html).
 
+# [1.31.0] - 2023-09-27
+
+## 🚀 Features
+
+### TLS client authentication for subgraph requests ([Issue #3414](https://github.com/apollographql/router/issues/3414))
+
+The router now supports TLS client authentication when connecting to subgraphs. It can be configured as follows:
+
+```yaml
+tls:
+  subgraph:
+    all:
+      client_authentication:
+        certificate_chain: 
+        key: 
+    # if configuring for a specific subgraph:
+    subgraphs:
+      # subgraph name
+      products:
+        client_authentication:
+          certificate_chain: 
+          key: 
+```
+
+Details on TLS client authentication can be found in the [documentation](https://www.apollographql.com/docs/router/configuration/overview#tls-client-authentication-for-subgraph-requests)
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3794
+
+### Added configuration to set redis request timeout ([Issue #3621](https://github.com/apollographql/router/issues/3621))
+
+We added configuration to override default timeout for Redis requests. Default timeout was also changed from 1ms to **2ms**.
+
+Here is an example to change the timeout for [Distributed APQ](https://www.apollographql.com/docs/router/configuration/distributed-caching#distributed-apq-caching) (an Enterprise Feature):
+```yaml
+apq:
+  router:
+    cache:
+      redis:
+        urls: ["redis://..."]
+        timeout: 5ms
+```
+
+By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3817
+
+### JSON encoding and decoding in Rhai ([PR #3785](https://github.com/apollographql/router/pull/3785))
+
+It is now possible to encode or decode JSON from Rhai scripts using `json::encode` and `json::decode`.
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3785
+
+### Supergraph coprocessor implementation ([PR #3647](https://github.com/apollographql/router/pull/3647))
+
+Coprocessors now support supergraph service interception.
+
+On the request side, the coprocessor payload can contain:
+- method
+- headers
+- body
+- context
+- sdl
+
+On the response side, the payload can contain:
+- status_code
+- headers
+- body
+- context
+- sdl
+
+The supergraph request body contains:
+- query
+- operation name
+- variables
+- extensions
+
+The supergraph response body contains:
+- label
+- data
+- errors
+- extensions
+
+When using `@defer` or subscriptions a supergraph response may contain multiple GraphQL responses. The coprocessor will be called for each response. Please refer to our [coprocessor documentation](https://www.apollographql.com/docs/router/customizations/coprocessor) for more information.
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3647
+
+### Adds support for the OpenTelemetry AWS X-Ray tracing propagator ([PR #3580](https://github.com/apollographql/router/pull/3580))
+
+This propagator helps propagate tracing information from upstream services (such as AWS load balancers) to downstream services. It also handles conversion between the X-Ray trace id format and OpenTelemetry span contexts.
+
+By [@scottmace](https://github.com/scottmace) in https://github.com/apollographql/router/pull/3580
+
+### HTTP/2 Cleartext protocol (H2C) support for subgraph connections ([Issue #3535](https://github.com/apollographql/router/issues/3535))
+
+The router can now connect to subgraphs over HTTP/2 Cleartext (H2C), which uses the HTTP/2 binary protocol directly over TCP **without TLS**, which is a mode of operation desired with some service mesh configurations (e.g., Istio, Envoy) where the value of added encryption is unnecessary. To activate it, set the `experimental_http2` option to `http2_only`.
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3852
+
+### Query plan cache warm-up improvements ([Issue #3704](https://github.com/apollographql/router/issues/3704), [Issue #3767](https://github.com/apollographql/router/issues/3767))
+
+The `warm_up_queries` option enables quicker schema updates by precomputing query plans for your most used cached queries and your persisted queries. When a new schema is loaded, a precomputed query plan for it may already be in the in-memory cache.
+
+We made a series of improvements to this feature to make it easier to use:
+* It is now active by default.
+* It warms up the cache with the 30% most used queries from previous cache.
+* The query cache percentage continues to be configurable, and it can be deactivated by setting it to 0.
+* The warm-up will now plan queries in random order to make sure that the work can be shared by multiple router instances using distributed caching.
+* Persisted queries are part of the warmed up queries.
+
+We also added histogram metrics for `apollo_router_query_planning_warmup_duration` and `apollo_router_schema_load_duration`. These metrics make it easier to track the time spent loading a new schema and planning queries in the warm-up phase. You can measure the query plan cache usage for both the in-memory-cache and distributed cache. This makes it easier to know how many entries are used as well as the cache hit rate.
+
+Here is what these metrics would look like in Prometheus:
+
+```
+# HELP apollo_router_query_planning_warmup_duration apollo_router_query_planning_warmup_duration
+# TYPE apollo_router_query_planning_warmup_duration histogram
+apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 1
+apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 1
+apollo_router_query_planning_warmup_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 1
+apollo_router_query_planning_warmup_duration_sum{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 0.022390619
+apollo_router_query_planning_warmup_duration_count{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version=""} 1
+# HELP apollo_router_schema_load_duration apollo_router_schema_load_duration
+# TYPE apollo_router_schema_load_duration histogram
+apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.05"} 8
+apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.1"} 8
+apollo_router_schema_load_duration_bucket{service_name="apollo-router",otel_scope_name="apollo/router",otel_scope_version="",le="0.25"} 8
+```
+
+You can get more information about operating the query plan cache and its warm-up phase in the [documentation](https://www.apollographql.com/docs/router/configuration/in-memory-caching#cache-warm-up)
+
+By [@Geal](https://github.com/Geal) in https://github.com/apollographql/router/pull/3815 https://github.com/apollographql/router/pull/3801 https://github.com/apollographql/router/pull/3767 https://github.com/apollographql/router/pull/3769 https://github.com/apollographql/router/pull/3770
+
+By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/3807
+
+## 🐛 Fixes
+
+### Fix error response on large number types in query transformations ([PR #3820](https://github.com/apollographql/router/pull/3820))
+
+This bug caused the router to reject operations where a large hardcoded integer was used as input for a Float field:
+
+```graphql
+# Schema
+type Query {
+    field(argument: Float): Int!
+}
+# Operation
+{
+    field(argument: 123456789123)
+}
+```
+
+This number is now correctly interpreted as a `Float`. This bug only affected hardcoded numbers, not numbers provided through variables.
+
+By [@goto-bus-stop](https://github.com/goto-bus-stop) in https://github.com/apollographql/router/pull/3820
+
+### Fix validation error with `ID` variable values overflowing 32-bit integers ([Issue #3873](https://github.com/apollographql/router/issues/3873))
+
+Input values for variables of type `ID` were previously validated as "either like a GraphQL `Int` or like a GraphQL `String`". GraphQL `Int` is specified as a signed 32-bit integer, such that values that overflow fail validation. Applying this range restriction to `ID` values was incorrect. Instead, validation for `ID` now accepts any JSON integer or JSON string value, so that IDs larger than 32 bits can be used.
+
+By [@SimonSapin](https://github.com/SimonSapin) in https://github.com/apollographql/router/pull/3896
+
+### Improve multi-cloud failover and error handling for Persisted Queries ([PR #3863](https://github.com/apollographql/router/pull/3863))
+
+Improves the resilience of the Persisted Queries feature to Uplink outages. This makes errors while fetching persisted query manifests from Uplink more visible.
+
+By [@glasser](https://github.com/glasser) in https://github.com/apollographql/router/pull/3863
+
+### Coprocessors: Discard content-length sent by coprocessors ([PR #3802](https://github.com/apollographql/router/pull/3802))
+
+The `content-length` of an HTTP response can only be computed when a router response is being sent.
+We now discard coprocessors `content-length` header to make sure the value is computed correctly.
+
+By [@o0Ignition0o](https://github.com/o0Ignition0o) in https://github.com/apollographql/router/pull/3802
+
+### Helm: If there are `extraLabels` add them to all resources ([PR #3622](https://github.com/apollographql/router/pull/3622))
+
+This extends the functionality of `extraLabels` so that, if they are defined, they will be templated for all resources created by the chart.
+
+Previously, they were only templated onto the `Deployment` resource.
+
+By [@garypen](https://github.com/garypen) and [@bjoernw](https://github.com/bjoernw) in https://github.com/apollographql/router/pull/3622
+
+## 📚 Documentation
+
+### Rhai documentation: remove incorrect statement about request.subgraph fields ([PR #3808](https://github.com/apollographql/router/pull/3808))
+
+It is possible to modify `request.subgraph` fields from a Rhai script, which is now correctly reflected in [Rhai documentation](https://www.apollographql.com/docs/router/customizations/rhai-api/#request-interface).
+
+By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/3808
+
+
+
 # [1.30.1] - 2023-09-22
 
 ## 🐛 Fixes
diff --git a/Cargo.lock b/Cargo.lock
index 113b86a69d..165dab1f22 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -264,7 +264,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router"
-version = "1.31.0-alpha.4"
+version = "1.31.0"
 dependencies = [
  "access-json",
  "anyhow",
@@ -411,7 +411,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.4"
+version = "1.31.0"
 dependencies = [
  "apollo-parser 0.6.2",
  "apollo-router",
@@ -427,7 +427,7 @@ dependencies = [
 
 [[package]]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.4"
+version = "1.31.0"
 dependencies = [
  "anyhow",
  "cargo-scaffold",
diff --git a/apollo-router-benchmarks/Cargo.toml b/apollo-router-benchmarks/Cargo.toml
index 65e513ba80..8c03e279d0 100644
--- a/apollo-router-benchmarks/Cargo.toml
+++ b/apollo-router-benchmarks/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-benchmarks"
-version = "1.31.0-alpha.4"
+version = "1.31.0"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml
index afab3bfe16..f6f2f3397c 100644
--- a/apollo-router-scaffold/Cargo.toml
+++ b/apollo-router-scaffold/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router-scaffold"
-version = "1.31.0-alpha.4"
+version = "1.31.0"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "Elastic-2.0"
diff --git a/apollo-router-scaffold/templates/base/Cargo.toml b/apollo-router-scaffold/templates/base/Cargo.toml
index 8e00a942bf..205692c26c 100644
--- a/apollo-router-scaffold/templates/base/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/Cargo.toml
@@ -22,7 +22,7 @@ apollo-router = { path ="{{integration_test}}apollo-router" }
 apollo-router = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
 # Note if you update these dependencies then also update xtask/Cargo.toml
-apollo-router = "1.31.0-alpha.4"
+apollo-router = "1.31.0"
 {{/if}}
 {{/if}}
 async-trait = "0.1.52"
diff --git a/apollo-router-scaffold/templates/base/xtask/Cargo.toml b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
index 30bc08f644..b731ddadeb 100644
--- a/apollo-router-scaffold/templates/base/xtask/Cargo.toml
+++ b/apollo-router-scaffold/templates/base/xtask/Cargo.toml
@@ -13,7 +13,7 @@ apollo-router-scaffold = { path ="{{integration_test}}apollo-router-scaffold" }
 {{#if branch}}
 apollo-router-scaffold = { git="https://github.com/apollographql/router.git", branch="{{branch}}" }
 {{else}}
-apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0-alpha.4" }
+apollo-router-scaffold = { git = "https://github.com/apollographql/router.git", tag = "v1.31.0" }
 {{/if}}
 {{/if}}
 anyhow = "1.0.58"
diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml
index b6dfc7ce39..1ff73d7563 100644
--- a/apollo-router/Cargo.toml
+++ b/apollo-router/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "apollo-router"
-version = "1.31.0-alpha.4"
+version = "1.31.0"
 authors = ["Apollo Graph, Inc. "]
 repository = "https://github.com/apollographql/router/"
 documentation = "https://docs.rs/apollo-router"
diff --git a/dockerfiles/tracing/docker-compose.datadog.yml b/dockerfiles/tracing/docker-compose.datadog.yml
index 5f12f785af..79d853c75c 100644
--- a/dockerfiles/tracing/docker-compose.datadog.yml
+++ b/dockerfiles/tracing/docker-compose.datadog.yml
@@ -3,7 +3,7 @@ services:
 
   apollo-router:
     container_name: apollo-router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.4
+    image: ghcr.io/apollographql/router:v1.31.0
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/datadog.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.jaeger.yml b/dockerfiles/tracing/docker-compose.jaeger.yml
index 19c83e15cd..9a6e59f1f8 100644
--- a/dockerfiles/tracing/docker-compose.jaeger.yml
+++ b/dockerfiles/tracing/docker-compose.jaeger.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     #build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.4
+    image: ghcr.io/apollographql/router:v1.31.0
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/jaeger.router.yaml:/etc/config/configuration.yaml
diff --git a/dockerfiles/tracing/docker-compose.zipkin.yml b/dockerfiles/tracing/docker-compose.zipkin.yml
index c6122bd9a7..8c95c95183 100644
--- a/dockerfiles/tracing/docker-compose.zipkin.yml
+++ b/dockerfiles/tracing/docker-compose.zipkin.yml
@@ -4,7 +4,7 @@ services:
   apollo-router:
     container_name: apollo-router
     build: ./router
-    image: ghcr.io/apollographql/router:v1.31.0-alpha.4
+    image: ghcr.io/apollographql/router:v1.31.0
     volumes:
       - ./supergraph.graphql:/etc/config/supergraph.graphql
       - ./router/zipkin.router.yaml:/etc/config/configuration.yaml
diff --git a/docs/source/containerization/docker.mdx b/docs/source/containerization/docker.mdx
index b0e93b2897..15add92d7d 100644
--- a/docs/source/containerization/docker.mdx
+++ b/docs/source/containerization/docker.mdx
@@ -11,7 +11,7 @@ The default behaviour of the router images is suitable for a quickstart or devel
 
 Note: The [docker documentation](https://docs.docker.com/engine/reference/run/) for the run command may be helpful when reading through the examples.
 
-Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0-alpha.4`
+Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.31.0`
 
 ## Override the configuration
 
diff --git a/docs/source/containerization/kubernetes.mdx b/docs/source/containerization/kubernetes.mdx
index 644eb6d9fa..eed4c546ff 100644
--- a/docs/source/containerization/kubernetes.mdx
+++ b/docs/source/containerization/kubernetes.mdx
@@ -13,7 +13,7 @@ import { Link } from 'gatsby';
 
 [Helm](https://helm.sh) is the package manager for kubernetes.
 
-There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0-alpha.4/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
+There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.31.0/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes.
 
 In both the following examples, we are using helm to install the router:
  - into namespace "router-deploy" (create namespace if it doesn't exist)
@@ -64,10 +64,10 @@ kind: ServiceAccount
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.4
+    helm.sh/chart: router-1.31.0
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.4"
+    app.kubernetes.io/version: "v1.31.0"
     app.kubernetes.io/managed-by: Helm
 ---
 # Source: router/templates/secret.yaml
@@ -76,10 +76,10 @@ kind: Secret
 metadata:
   name: "release-name-router"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.4
+    helm.sh/chart: router-1.31.0
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.4"
+    app.kubernetes.io/version: "v1.31.0"
     app.kubernetes.io/managed-by: Helm
 data:
   managedFederationApiKey: "UkVEQUNURUQ="
@@ -90,10 +90,10 @@ kind: ConfigMap
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.4
+    helm.sh/chart: router-1.31.0
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.4"
+    app.kubernetes.io/version: "v1.31.0"
     app.kubernetes.io/managed-by: Helm
 data:
   configuration.yaml: |
@@ -117,10 +117,10 @@ kind: Service
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.4
+    helm.sh/chart: router-1.31.0
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.4"
+    app.kubernetes.io/version: "v1.31.0"
     app.kubernetes.io/managed-by: Helm
 spec:
   type: ClusterIP
@@ -143,10 +143,10 @@ kind: Deployment
 metadata:
   name: release-name-router
   labels:
-    helm.sh/chart: router-1.31.0-alpha.4
+    helm.sh/chart: router-1.31.0
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.4"
+    app.kubernetes.io/version: "v1.31.0"
     app.kubernetes.io/managed-by: Helm
   
   annotations:
@@ -174,7 +174,7 @@ spec:
         - name: router
           securityContext:
             {}
-          image: "ghcr.io/apollographql/router:v1.31.0-alpha.4"
+          image: "ghcr.io/apollographql/router:v1.31.0"
           imagePullPolicy: IfNotPresent
           args:
             - --hot-reload
@@ -226,10 +226,10 @@ kind: Pod
 metadata:
   name: "release-name-router-test-connection"
   labels:
-    helm.sh/chart: router-1.31.0-alpha.4
+    helm.sh/chart: router-1.31.0
     app.kubernetes.io/name: router
     app.kubernetes.io/instance: release-name
-    app.kubernetes.io/version: "v1.31.0-alpha.4"
+    app.kubernetes.io/version: "v1.31.0"
     app.kubernetes.io/managed-by: Helm
   annotations:
     "helm.sh/hook": test
diff --git a/docs/source/federation-version-support.mdx b/docs/source/federation-version-support.mdx
index a44f7f12ef..f72855fb1f 100644
--- a/docs/source/federation-version-support.mdx
+++ b/docs/source/federation-version-support.mdx
@@ -27,7 +27,15 @@ The table below shows which version of federation each router release is compile
     
     
         
-            v1.29.1 and later (see latest releases)
+            v1.30.0 and later (see latest releases)
+        
+        
+            2.5.4
+        
+    
+    
+        
+            ️v1.29.1
         
         
             2.5.3
diff --git a/helm/chart/router/Chart.yaml b/helm/chart/router/Chart.yaml
index 1c70ef5a8a..eddbddcde9 100644
--- a/helm/chart/router/Chart.yaml
+++ b/helm/chart/router/Chart.yaml
@@ -20,10 +20,10 @@ type: application
 # so it matches the shape of our release process and release automation.
 # By proxy of that decision, this version uses SemVer 2.0.0, though the prefix
 # of "v" is not included.
-version: 1.31.0-alpha.4
+version: 1.31.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.31.0-alpha.4"
+appVersion: "v1.31.0"
diff --git a/helm/chart/router/README.md b/helm/chart/router/README.md
index 939ce9ec39..7aec5b87f5 100644
--- a/helm/chart/router/README.md
+++ b/helm/chart/router/README.md
@@ -2,7 +2,7 @@
 
 [router](https://github.com/apollographql/router) Rust Graph Routing runtime for Apollo Federation
 
-![Version: 1.31.0-alpha.4](https://img.shields.io/badge/Version-1.31.0--alpha.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0-alpha.4](https://img.shields.io/badge/AppVersion-v1.31.0--alpha.4-informational?style=flat-square)
+![Version: 1.31.0](https://img.shields.io/badge/Version-1.31.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31.0](https://img.shields.io/badge/AppVersion-v1.31.0-informational?style=flat-square)
 
 ## Prerequisites
 
@@ -11,7 +11,7 @@
 ## Get Repo Info
 
 ```console
-helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.4
+helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0
 ```
 
 ## Install Chart
@@ -19,7 +19,7 @@ helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.
 **Important:** only helm3 is supported
 
 ```console
-helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0-alpha.4 --values my-values.yaml
+helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.31.0 --values my-values.yaml
 ```
 
 _See [configuration](#configuration) below._
@@ -94,4 +94,4 @@ helm show values oci://ghcr.io/apollographql/helm-charts/router
 | virtualservice.enabled | bool | `false` |  |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)
+Autogenerated from chart metadata using [helm-docs v1.11.2](https://github.com/norwoodj/helm-docs/releases/v1.11.2)
diff --git a/licenses.html b/licenses.html
index a77c5f6ba3..e889944a23 100644
--- a/licenses.html
+++ b/licenses.html
@@ -12058,9 +12058,9 @@ 
      Used by:
      
                 
-                
      Copyright 2021 Apollo Graph, Inc.
+                
      Elastic License 2.0
 
-Elastic License 2.0
+URL: https://www.elastic.co/licensing/elastic-license
 
 ## Acceptance
 
@@ -12151,8 +12151,6 @@ 
      Used by:
      
 **use** means anything you do with the software requiring one of your licenses.
 
 **trademark** means trademarks, service marks, and similar rights.
-
---------------------------------------------------------------------------------
 
      
             
             
    • 
diff --git a/scripts/install.sh b/scripts/install.sh
index 743f1ee55f..ec5ed276b2 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -11,7 +11,7 @@ BINARY_DOWNLOAD_PREFIX="https://github.com/apollographql/router/releases/downloa
 
 # Router version defined in apollo-router's Cargo.toml
 # Note: Change this line manually during the release steps.
-PACKAGE_VERSION="v1.31.0-alpha.4"
+PACKAGE_VERSION="v1.31.0"
 
 download_binary() {
     downloader --check