Revert "feat: add dedupe capability on all IO operations (#2922)"

This reverts commit 1cacf4a. # Conflicts: # tests/core/snapshots/apollo-federation-entities-batch.md_1.snap # tests/core/snapshots/apollo-federation-entities.md_1.snap
tailcallhq · Sep 29, 2024 · d56f846 · d56f846
1 parent a302df1
commit d56f846
Show file tree

Hide file tree

Showing 32 changed files with 126 additions and 270 deletions.
diff --git a/generated/.tailcallrc.graphql b/generated/.tailcallrc.graphql
@@ -35,13 +35,6 @@ directive @cache(
 Provides the ability to refer to multiple fields in the Query or Mutation root.
 """
 directive @call(
-  """
-  Enables deduplication of IO operations to enhance performance.This flag prevents 
-  duplicate IO requests from being executed concurrently, reducing resource load. Caution: 
-  May lead to issues with APIs that expect unique results for identical inputs, such 
-  as nonce-based APIs.
-  """
-  dedupe: Boolean
   """
   Steps are composed together to form a call. If you have multiple steps, the output 
   of the previous step is passed as input to the next step.
@@ -78,13 +71,6 @@ directive @graphQL(
   """
   batch: Boolean!
   """
-  Enables deduplication of IO operations to enhance performance.This flag prevents 
-  duplicate IO requests from being executed concurrently, reducing resource load. Caution: 
-  May lead to issues with APIs that expect unique results for identical inputs, such 
-  as nonce-based APIs.
-  """
-  dedupe: Boolean
-  """
   The headers parameter allows you to customize the headers of the GraphQL request 
   made by the `@graphQL` operator. It is used by specifying a key-value map of header 
   names and their values.
@@ -125,13 +111,6 @@ directive @grpc(
   """
   body: JSON
   """
-  Enables deduplication of IO operations to enhance performance.This flag prevents 
-  duplicate IO requests from being executed concurrently, reducing resource load. Caution: 
-  May lead to issues with APIs that expect unique results for identical inputs, such 
-  as nonce-based APIs.
-  """
-  dedupe: Boolean
-  """
   The `headers` parameter allows you to customize the headers of the HTTP request made 
   by the `@grpc` operator. It is used by specifying a key-value map of header names 
   and their values. Note: content-type is automatically set to application/grpc
@@ -169,13 +148,6 @@ directive @http(
   """
   body: String
   """
-  Enables deduplication of IO operations to enhance performance.This flag prevents 
-  duplicate IO requests from being executed concurrently, reducing resource load. Caution: 
-  May lead to issues with APIs that expect unique results for identical inputs, such 
-  as nonce-based APIs.
-  """
-  dedupe: Boolean
-  """
   The `encoding` parameter specifies the encoding of the request body. It can be `ApplicationJson` 
   or `ApplicationXWwwFormUrlEncoded`. @default `ApplicationJson`.
   """
@@ -281,6 +253,13 @@ directive @server(
   introducing latency and complicating debugging. Use judiciously. @default `false`.
   """
   batchRequests: Boolean
+  """
+  Enables deduplication of IO operations to enhance performance.This flag prevents 
+  duplicate IO requests from being executed concurrently, reducing resource load. Caution: 
+  May lead to issues with APIs that expect unique results for identical inputs, such 
+  as nonce-based APIs.
+  """
+  dedupe: Boolean
   enableJIT: Boolean
   """
   `globalResponseTimeout` sets the maximum query duration before termination, acting 
@@ -761,13 +740,6 @@ input GraphQL {
   """
   batch: Boolean!
   """
-  Enables deduplication of IO operations to enhance performance.This flag prevents 
-  duplicate IO requests from being executed concurrently, reducing resource load. Caution: 
-  May lead to issues with APIs that expect unique results for identical inputs, such 
-  as nonce-based APIs.
-  """
-  dedupe: Boolean
-  """
   The headers parameter allows you to customize the headers of the GraphQL request 
   made by the `@graphQL` operator. It is used by specifying a key-value map of header 
   names and their values.
@@ -808,13 +780,6 @@ input Grpc {
   """
   body: JSON
   """
-  Enables deduplication of IO operations to enhance performance.This flag prevents 
-  duplicate IO requests from being executed concurrently, reducing resource load. Caution: 
-  May lead to issues with APIs that expect unique results for identical inputs, such 
-  as nonce-based APIs.
-  """
-  dedupe: Boolean
-  """
   The `headers` parameter allows you to customize the headers of the HTTP request made 
   by the `@grpc` operator. It is used by specifying a key-value map of header names 
   and their values. Note: content-type is automatically set to application/grpc
@@ -852,13 +817,6 @@ input Http {
   """
   body: String
   """
-  Enables deduplication of IO operations to enhance performance.This flag prevents 
-  duplicate IO requests from being executed concurrently, reducing resource load. Caution: 
-  May lead to issues with APIs that expect unique results for identical inputs, such 
-  as nonce-based APIs.
-  """
-  dedupe: Boolean
-  """
   The `encoding` parameter specifies the encoding of the request body. It can be `ApplicationJson` 
   or `ApplicationXWwwFormUrlEncoded`. @default `ApplicationJson`.
   """

diff --git a/generated/.tailcallrc.schema.json b/generated/.tailcallrc.schema.json
@@ -229,13 +229,6 @@
         "steps"
       ],
       "properties": {
-        "dedupe": {
-          "description": "Enables deduplication of IO operations to enhance performance.\n\nThis flag prevents duplicate IO requests from being executed concurrently, reducing resource load. Caution: May lead to issues with APIs that expect unique results for identical inputs, such as nonce-based APIs.",
-          "type": [
-            "boolean",
-            "null"
-          ]
-        },
         "steps": {
           "description": "Steps are composed together to form a call. If you have multiple steps, the output of the previous step is passed as input to the next step.",
           "type": "array",
@@ -548,13 +541,6 @@
           "description": "If the upstream GraphQL server supports request batching, you can specify the 'batch' argument to batch several requests into a single batch request.\n\nMake sure you have also specified batch settings to the `@upstream` and to the `@graphQL` operator.",
           "type": "boolean"
         },
-        "dedupe": {
-          "description": "Enables deduplication of IO operations to enhance performance.\n\nThis flag prevents duplicate IO requests from being executed concurrently, reducing resource load. Caution: May lead to issues with APIs that expect unique results for identical inputs, such as nonce-based APIs.",
-          "type": [
-            "boolean",
-            "null"
-          ]
-        },
         "headers": {
           "description": "The headers parameter allows you to customize the headers of the GraphQL request made by the `@graphQL` operator. It is used by specifying a key-value map of header names and their values.",
           "type": "array",
@@ -593,13 +579,6 @@
         "body": {
           "description": "This refers to the arguments of your gRPC call. You can pass it as a static object or use Mustache template for dynamic parameters. These parameters will be added in the body in `protobuf` format."
         },
-        "dedupe": {
-          "description": "Enables deduplication of IO operations to enhance performance.\n\nThis flag prevents duplicate IO requests from being executed concurrently, reducing resource load. Caution: May lead to issues with APIs that expect unique results for identical inputs, such as nonce-based APIs.",
-          "type": [
-            "boolean",
-            "null"
-          ]
-        },
         "headers": {
           "description": "The `headers` parameter allows you to customize the headers of the HTTP request made by the `@grpc` operator. It is used by specifying a key-value map of header names and their values. Note: content-type is automatically set to application/grpc",
           "type": "array",
@@ -690,13 +669,6 @@
             "null"
           ]
         },
-        "dedupe": {
-          "description": "Enables deduplication of IO operations to enhance performance.\n\nThis flag prevents duplicate IO requests from being executed concurrently, reducing resource load. Caution: May lead to issues with APIs that expect unique results for identical inputs, such as nonce-based APIs.",
-          "type": [
-            "boolean",
-            "null"
-          ]
-        },
         "encoding": {
           "description": "The `encoding` parameter specifies the encoding of the request body. It can be `ApplicationJson` or `ApplicationXWwwFormUrlEncoded`. @default `ApplicationJson`.",
           "allOf": [
@@ -1046,6 +1018,13 @@
             "null"
           ]
         },
+        "dedupe": {
+          "description": "Enables deduplication of IO operations to enhance performance.\n\nThis flag prevents duplicate IO requests from being executed concurrently, reducing resource load. Caution: May lead to issues with APIs that expect unique results for identical inputs, such as nonce-based APIs.",
+          "type": [
+            "boolean",
+            "null"
+          ]
+        },
         "enableJIT": {
           "type": [
             "boolean",

diff --git a/src/core/app_context.rs b/src/core/app_context.rs
@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use async_graphql::dynamic::{self, DynamicRequest};
 use async_graphql_value::ConstValue;
+use hyper::body::Bytes;
 
 use crate::core::async_graphql_hyper::OperationId;
 use crate::core::auth::context::GlobalAuthContext;
@@ -10,7 +11,7 @@ use crate::core::data_loader::{DataLoader, DedupeResult};
 use crate::core::graphql::GraphqlDataLoader;
 use crate::core::grpc;
 use crate::core::grpc::data_loader::GrpcDataLoader;
-use crate::core::http::{DataLoaderRequest, HttpDataLoader};
+use crate::core::http::{DataLoaderRequest, HttpDataLoader, Response};
 use crate::core::ir::model::{DataLoaderId, IoId, IO, IR};
 use crate::core::ir::Error;
 use crate::core::rest::{Checked, EndpointSet};
@@ -26,7 +27,7 @@ pub struct AppContext {
     pub endpoints: EndpointSet<Checked>,
     pub auth_ctx: Arc<GlobalAuthContext>,
     pub dedupe_handler: Arc<DedupeResult<IoId, ConstValue, Error>>,
-    pub dedupe_operation_handler: DedupeResult<OperationId, Arc<async_graphql::Response>, Error>,
+    pub dedupe_operation_handler: DedupeResult<OperationId, Response<Bytes>, Error>,
 }
 
 impl AppContext {
@@ -47,15 +48,9 @@ impl AppContext {
                         expr.modify(&mut |expr| match expr {
                             IR::IO(io) => match io {
                                 IO::Http {
-                                    req_template,
-                                    group_by,
-                                    http_filter,
-                                    is_list,
-                                    dedupe,
-                                    ..
+                                    req_template, group_by, http_filter, is_list, ..
                                 } => {
                                     let is_list = *is_list;
-                                    let dedupe = *dedupe;
                                     let data_loader = HttpDataLoader::new(
                                         runtime.clone(),
                                         group_by.clone(),
@@ -69,16 +64,14 @@ impl AppContext {
                                         dl_id: Some(DataLoaderId::new(http_data_loaders.len())),
                                         http_filter: http_filter.clone(),
                                         is_list,
-                                        dedupe,
                                     }));
 
                                     http_data_loaders.push(data_loader);
 
                                     result
                                 }
 
-                                IO::GraphQL { req_template, field_name, batch, dedupe, .. } => {
-                                    let dedupe = *dedupe;
+                                IO::GraphQL { req_template, field_name, batch, .. } => {
                                     let graphql_data_loader =
                                         GraphqlDataLoader::new(runtime.clone(), *batch)
                                             .into_data_loader(
@@ -90,16 +83,14 @@ impl AppContext {
                                         field_name: field_name.clone(),
                                         batch: *batch,
                                         dl_id: Some(DataLoaderId::new(gql_data_loaders.len())),
-                                        dedupe,
                                     }));
 
                                     gql_data_loaders.push(graphql_data_loader);
 
                                     result
                                 }
 
-                                IO::Grpc { req_template, group_by, dedupe, .. } => {
-                                    let dedupe = *dedupe;
+                                IO::Grpc { req_template, group_by, .. } => {
                                     let data_loader = GrpcDataLoader {
                                         runtime: runtime.clone(),
                                         operation: req_template.operation.clone(),
@@ -113,7 +104,6 @@ impl AppContext {
                                         req_template: req_template.clone(),
                                         group_by: group_by.clone(),
                                         dl_id: Some(DataLoaderId::new(grpc_data_loaders.len())),
-                                        dedupe,
                                     }));
 
                                     grpc_data_loaders.push(data_loader);

diff --git a/src/core/blueprint/operators/graphql.rs b/src/core/blueprint/operators/graphql.rs
@@ -71,8 +71,7 @@ pub fn compile_graphql(
     .map(|req_template| {
         let field_name = graphql.name.clone();
         let batch = graphql.batch;
-        let dedupe = graphql.dedupe.unwrap_or_default();
-        IR::IO(IO::GraphQL { req_template, field_name, batch, dl_id: None, dedupe })
+        IR::IO(IO::GraphQL { req_template, field_name, batch, dl_id: None })
     })
 }
 

diff --git a/src/core/blueprint/operators/grpc.rs b/src/core/blueprint/operators/grpc.rs
@@ -160,7 +160,6 @@ pub fn compile_grpc(inputs: CompileGrpc) -> Valid<IR, String> {
     let field = inputs.field;
     let grpc = inputs.grpc;
     let validate_with_schema = inputs.validate_with_schema;
-    let dedupe = grpc.dedupe.unwrap_or_default();
 
     Valid::from(GrpcMethod::try_from(grpc.method.as_str()))
         .and_then(|method| {
@@ -202,10 +201,9 @@ pub fn compile_grpc(inputs: CompileGrpc) -> Valid<IR, String> {
                     req_template,
                     group_by: Some(GroupBy::new(grpc.batch_key.clone(), None)),
                     dl_id: None,
-                    dedupe,
                 })
             } else {
-                IR::IO(IO::Grpc { req_template, group_by: None, dl_id: None, dedupe })
+                IR::IO(IO::Grpc { req_template, group_by: None, dl_id: None })
             }
         })
 }

diff --git a/src/core/blueprint/operators/http.rs b/src/core/blueprint/operators/http.rs
@@ -13,8 +13,6 @@ pub fn compile_http(
     http: &config::Http,
     is_list: bool,
 ) -> Valid<IR, String> {
-    let dedupe = http.dedupe.unwrap_or_default();
-
     Valid::<(), String>::fail("GroupBy is only supported for GET requests".to_string())
         .when(|| !http.batch_key.is_empty() && http.method != Method::GET)
         .and(
@@ -83,7 +81,6 @@ pub fn compile_http(
                     dl_id: None,
                     http_filter,
                     is_list,
-                    dedupe,
                 })
             } else {
                 IR::IO(IO::Http {
@@ -92,7 +89,6 @@ pub fn compile_http(
                     dl_id: None,
                     http_filter,
                     is_list,
-                    dedupe,
                 })
             }
         })

diff --git a/src/core/blueprint/server.rs b/src/core/blueprint/server.rs
@@ -37,6 +37,7 @@ pub struct Server {
     pub cors: Option<Cors>,
     pub experimental_headers: HashSet<HeaderName>,
     pub auth: Option<Auth>,
+    pub dedupe: bool,
     pub routes: Routes,
 }
 
@@ -153,6 +154,7 @@ impl TryFrom<crate::core::config::ConfigModule> for Server {
                         script,
                         cors,
                         auth,
+                        dedupe: config_server.get_dedupe(),
                         routes: config_server.get_routes(),
                     }
                 },