From d188ce89b26962afb31b78a15aeb8d73b69aa703 Mon Sep 17 00:00:00 2001 From: Panagiotis Date: Tue, 13 Aug 2024 17:59:07 +0300 Subject: [PATCH 1/8] Enable to fetch all dependencies from reflection when source is reflection url --- src/core/proto_reader/fetch.rs | 11 ++++-- src/core/proto_reader/reader.rs | 59 +++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/src/core/proto_reader/fetch.rs b/src/core/proto_reader/fetch.rs index 9bb8a70f37..36ff85617e 100644 --- a/src/core/proto_reader/fetch.rs +++ b/src/core/proto_reader/fetch.rs @@ -110,7 +110,14 @@ impl GrpcReflection { .execute(json!({"file_containing_symbol": service})) .await?; - request_proto(resp).await + request_proto(resp) + } + + /// Makes `Get File` request to grpc reflection server + pub async fn get_file(&self, file_path: &str) -> Result { + let resp = self.execute(json!({"file_by_filename": file_path})).await?; + + request_proto(resp) } async fn execute(&self, body: serde_json::Value) -> Result { @@ -158,7 +165,7 @@ impl GrpcReflection { } /// For extracting `FileDescriptorProto` from `CustomResponse` -async fn request_proto(response: ReflectionResponse) -> Result { +fn request_proto(response: ReflectionResponse) -> Result { let file_descriptor_resp = response .file_descriptor_response .context("Expected fileDescriptorResponse but found none")?; diff --git a/src/core/proto_reader/reader.rs b/src/core/proto_reader/reader.rs index dd13d29e1e..55f28d072d 100644 --- a/src/core/proto_reader/reader.rs +++ b/src/core/proto_reader/reader.rs @@ -1,8 +1,10 @@ use std::collections::{HashMap, VecDeque}; use std::path::{Path, PathBuf}; +use std::sync::Arc; use anyhow::Context; -use futures_util::future::join_all; +use futures_util::future::{join_all, BoxFuture}; +use futures_util::FutureExt; use prost_reflect::prost_types::{FileDescriptorProto, FileDescriptorSet}; use protox::file::{FileResolver, GoogleFileResolver}; @@ -29,7 +31,7 @@ impl ProtoReader { /// Fetches proto files from a grpc server (grpc reflection) pub async fn fetch>(&self, url: T) -> anyhow::Result> { - let grpc_reflection = GrpcReflection::new(url.as_ref(), self.runtime.clone()); + let grpc_reflection = Arc::new(GrpcReflection::new(url.as_ref(), self.runtime.clone())); let mut proto_metadata = vec![]; let service_list = grpc_reflection.list_all_files().await?; @@ -39,7 +41,9 @@ impl ProtoReader { } let file_descriptor_proto = grpc_reflection.get_by_service(&service).await?; Self::check_package(&file_descriptor_proto)?; - let descriptors = self.resolve(file_descriptor_proto, None).await?; + let descriptors = self + .reflection_resolve(grpc_reflection.clone(), file_descriptor_proto) + .await?; let metadata = ProtoMetadata { descriptor_set: FileDescriptorSet { file: descriptors }, path: url.as_ref().to_string(), @@ -64,7 +68,7 @@ impl ProtoReader { Self::check_package(&file_read)?; let descriptors = self - .resolve(file_read, PathBuf::from(path.as_ref()).parent()) + .file_resolve(file_read, PathBuf::from(path.as_ref()).parent()) .await?; let metadata = ProtoMetadata { descriptor_set: FileDescriptorSet { file: descriptors }, @@ -73,12 +77,15 @@ impl ProtoReader { Ok(metadata) } - /// Performs BFS to import all nested proto files - async fn resolve( + /// Used as a helper file to resolve dependencies proto files + async fn resolve_dependencies( &self, parent_proto: FileDescriptorProto, - parent_path: Option<&Path>, - ) -> anyhow::Result> { + resolve_fn: F, + ) -> anyhow::Result> + where + F: Fn(&str) -> BoxFuture<'_, anyhow::Result>, + { let mut descriptors: HashMap = HashMap::new(); let mut queue = VecDeque::new(); queue.push_back(parent_proto.clone()); @@ -87,7 +94,7 @@ impl ProtoReader { let futures: Vec<_> = file .dependency .iter() - .map(|import| self.read_proto(import, parent_path)) + .map(|import| resolve_fn(import)) .collect(); let results = join_all(futures).await; @@ -108,6 +115,38 @@ impl ProtoReader { Ok(descriptors_vec) } + /// Used to resolve dependencies proto files using file reader + async fn file_resolve( + &self, + parent_proto: FileDescriptorProto, + parent_path: Option<&Path>, + ) -> anyhow::Result> { + let parent_path = parent_path.map(|p| p.to_path_buf()); + + self.resolve_dependencies(parent_proto, move |import| { + let parent_path = parent_path.clone(); + let self_ref = + ProtoReader { reader: self.reader.clone(), runtime: self.runtime.clone() }; + + async move { self_ref.read_proto(import, parent_path.as_deref()).await }.boxed() + }) + .await + } + + /// Used to resolve dependencies proto files using reflection + async fn reflection_resolve( + &self, + grpc_reflection: Arc, + parent_proto: FileDescriptorProto, + ) -> anyhow::Result> { + let grpc_reflection = Arc::clone(&grpc_reflection); + self.resolve_dependencies(parent_proto, move |file| { + let grpc_reflection = Arc::clone(&grpc_reflection); + async move { grpc_reflection.get_file(file).await }.boxed() + }) + .await + } + /// Tries to load well-known google proto files and if not found uses normal /// file and http IO to resolve them async fn read_proto>( @@ -180,7 +219,7 @@ mod test_proto_config { let reader = ProtoReader::init(ResourceReader::::cached(runtime.clone()), runtime); let file_descriptors = reader - .resolve(reader.read_proto(&test_file, None).await?, Some(test_dir)) + .file_resolve(reader.read_proto(&test_file, None).await?, Some(test_dir)) .await?; for file in file_descriptors .iter() From d93372a9974045e139ada2ad4d22af06845eaf81 Mon Sep 17 00:00:00 2001 From: Panagiotis Date: Wed, 14 Aug 2024 14:36:12 +0300 Subject: [PATCH 2/8] Add working test --- src/core/proto_reader/fetch.rs | 38 ++++++++++++++++++ .../proto_reader/fixtures/descriptor_b64.txt | 2 +- src/core/proto_reader/fixtures/dto_b64.txt | 1 + tailcall-fixtures/README.md | 9 +++++ ...ws-list-services.bin => list-services.bin} | Bin .../reflection/news-service-descriptor.bin | Bin 1373 -> 0 bytes .../fixtures/grpc/reflection/news-service.bin | Bin 0 -> 808 bytes .../fixtures/grpc/reflection/news_dto.bin | Bin 0 -> 1142 bytes .../grpc/reflection/protobuf_empty.bin | Bin 0 -> 2409 bytes .../fixtures/protobuf/news.proto | 23 +---------- .../fixtures/protobuf/news_dto.proto | 25 ++++++++++++ .../snapshots/grpc-reflection.md_client.snap | 7 ++++ .../snapshots/grpc-reflection.md_merged.snap | 7 ++++ tests/execution/grpc-reflection.md | 28 ++++++++++++- 14 files changed, 116 insertions(+), 24 deletions(-) create mode 100644 src/core/proto_reader/fixtures/dto_b64.txt rename tailcall-fixtures/fixtures/grpc/reflection/{news-list-services.bin => list-services.bin} (100%) delete mode 100644 tailcall-fixtures/fixtures/grpc/reflection/news-service-descriptor.bin create mode 100644 tailcall-fixtures/fixtures/grpc/reflection/news-service.bin create mode 100644 tailcall-fixtures/fixtures/grpc/reflection/news_dto.bin create mode 100644 tailcall-fixtures/fixtures/grpc/reflection/protobuf_empty.bin create mode 100644 tailcall-fixtures/fixtures/protobuf/news_dto.proto diff --git a/src/core/proto_reader/fetch.rs b/src/core/proto_reader/fetch.rs index 36ff85617e..773b21faa2 100644 --- a/src/core/proto_reader/fetch.rs +++ b/src/core/proto_reader/fetch.rs @@ -203,6 +203,16 @@ mod grpc_fetch { BASE64_STANDARD.decode(bytes).unwrap() } + fn get_dto_file_descriptor() -> Vec { + let mut path = PathBuf::from(file!()); + path.pop(); + path.push("fixtures/dto_b64.txt"); + + let bytes = std::fs::read(path).unwrap(); + + BASE64_STANDARD.decode(bytes).unwrap() + } + fn start_mock_server() -> httpmock::MockServer { httpmock::MockServer::start() } @@ -235,6 +245,34 @@ mod grpc_fetch { Ok(()) } + #[tokio::test] + async fn test_dto_file() -> Result<()> { + let server = start_mock_server(); + + let http_reflection_file_mock = server.mock(|when, then| { + when.method(httpmock::Method::POST) + .path("/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo") + .body("\0\0\0\0\u{10}\u{1a}\u{0e}news_dto.proto"); + then.status(200).body(get_dto_file_descriptor()); + }); + + let grpc_reflection = GrpcReflection::new( + format!("http://localhost:{}", server.port()), + crate::core::runtime::test::init(None), + ); + + let runtime = crate::core::runtime::test::init(None); + let resp = grpc_reflection.get_file("news_dto.proto").await?; + + let content = runtime.file.read(tailcall_fixtures::protobuf::NEWS_DTO).await?; + let expected = protox_parse::parse("news_dto.proto", &content)?; + + assert_eq!(expected.name(), resp.name()); + + http_reflection_file_mock.assert(); + Ok(()) + } + #[tokio::test] async fn test_resp_list_all() -> Result<()> { let server = start_mock_server(); diff --git a/src/core/proto_reader/fixtures/descriptor_b64.txt b/src/core/proto_reader/fixtures/descriptor_b64.txt index 8f93ea0ced..b82b3465bd 100644 --- a/src/core/proto_reader/fixtures/descriptor_b64.txt +++ b/src/core/proto_reader/fixtures/descriptor_b64.txt @@ -1 +1 @@ -AAAABVgSEiIQbmV3cy5OZXdzU2VydmljZSLBCgq+CgoKbmV3cy5wcm90bxIEbmV3cxobZ29vZ2xlL3Byb3RvYnVmL2VtcHR5LnByb3RvIkIKBE5ld3MSCgoCaWQYASABKAUSDQoFdGl0bGUYAiABKAkSDAoEYm9keRgDIAEoCRIRCglwb3N0SW1hZ2UYBCABKAkiFAoGTmV3c0lkEgoKAmlkGAEgASgFIiMKDk11bHRpcGxlTmV3c0lkEhEKA2lkcxgBIAMyBk5ld3NJZCIcCghOZXdzTGlzdBIQCgRuZXdzGAEgAzIETmV3czLeAQoLTmV3c1NlcnZpY2USLQoKR2V0QWxsTmV3cxIVZ29vZ2xlLnByb3RvYnVmLkVtcHR5GghOZXdzTGlzdBIXCgdHZXROZXdzEgZOZXdzSWQaBE5ld3MSKwoPR2V0TXVsdGlwbGVOZXdzEg5NdWx0aXBsZU5ld3NJZBoITmV3c0xpc3QSKwoKRGVsZXRlTmV3cxIGTmV3c0lkGhVnb29nbGUucHJvdG9idWYuRW1wdHkSFgoIRWRpdE5ld3MSBE5ld3MaBE5ld3MSFQoHQWRkTmV3cxIETmV3cxoETmV3c0qGBwoGEgQAACABCggKAQISAwQADQoJCgIDABIDAgAlCgoKAgQAEgQGAAsBCgoKAwQAARIDBggMCgsKBAQAAgASAwcEEQoMCgUEAAIAARIDBwoMCgwKBQQAAgADEgMHDxAKDAoFBAACAAUSAwcECQoLCgQEAAIBEgMIBBUKDAoFBAACAQESAwgLEAoMCgUEAAIBAxIDCBMUCgwKBQQAAgEFEgMIBAoKCwoEBAACAhIDCQQUCgwKBQQAAgIBEgMJCw8KDAoFBAACAgMSAwkSEwoMCgUEAAICBRIDCQQKCgsKBAQAAgMSAwoEGQoMCgUEAAIDARIDCgsUCgwKBQQAAgMDEgMKFxgKDAoFBAACAwUSAwoECgoKCgIEARIEFgAYAQoKCgMEAQESAxYIDgoLCgQEAQIAEgMXBBEKDAoFBAECAAESAxcKDAoMCgUEAQIAAxIDFw8QCgwKBQQBAgAFEgMXBAkKCgoCBAISBBoAHAEKCgoDBAIBEgMaCBYKCwoEBAICABIDGwQcCgwKBQQCAgABEgMbFBcKDAoFBAICAAMSAxsaGwoMCgUEAgIABBIDGwQMCgwKBQQCAgAGEgMbDRMKCgoCBAMSBB4AIAEKCgoDBAMBEgMeCBAKCwoEBAMCABIDHwMaCgwKBQQDAgABEgMfERUKDAoFBAMCAAMSAx8YGQoMCgUEAwIABBIDHwMLCgwKBQQDAgAGEgMfDBAKCgoCBgASBA0AFAEKCgoDBgABEgMNCBMKCwoEBgACABIDDgRACgwKBQYAAgABEgMOCBIKDAoFBgACAAISAw4UKQoMCgUGAAIAAxIDDjQ8CgsKBAYAAgESAw8EKgoMCgUGAAIBARIDDwgPCgwKBQYAAgECEgMPERcKDAoFBgACAQMSAw8iJgoLCgQGAAICEgMQBD4KDAoFBgACAgESAxAIFwoMCgUGAAICAhIDEBknCgwKBQYAAgIDEgMQMjoKCwoEBgACAxIDEQQ+CgwKBQYAAgMBEgMRCBIKDAoFBgACAwISAxEUGgoMCgUGAAIDAxIDESU6CgsKBAYAAgQSAxIEKQoMCgUGAAIEARIDEggQCgwKBQYAAgQCEgMSEhYKDAoFBgACBAMSAxIhJQoLCgQGAAIFEgMTBCgKDAoFBgACBQESAxMIDwoMCgUGAAIFAhIDExEVCgwKBQYAAgUDEgMTICQKCAoBDBIDAAASYgZwcm90bzM= \ No newline at end of file +AAAAAyMSEiIQbmV3cy5OZXdzU2VydmljZSKMBgqJBgoKbmV3cy5wcm90bxIEbmV3cxoObmV3c19kdG8ucHJvdG8aG2dvb2dsZS9wcm90b2J1Zi9lbXB0eS5wcm90bzKoAgoLTmV3c1NlcnZpY2USNgoKR2V0QWxsTmV3cxIWLmdvb2dsZS5wcm90b2J1Zi5FbXB0eRoOLm5ld3MuTmV3c0xpc3QiABIlCgdHZXROZXdzEgwubmV3cy5OZXdzSWQaCi5uZXdzLk5ld3MiABI5Cg9HZXRNdWx0aXBsZU5ld3MSFC5uZXdzLk11bHRpcGxlTmV3c0lkGg4ubmV3cy5OZXdzTGlzdCIAEjQKCkRlbGV0ZU5ld3MSDC5uZXdzLk5ld3NJZBoWLmdvb2dsZS5wcm90b2J1Zi5FbXB0eSIAEiQKCEVkaXROZXdzEgoubmV3cy5OZXdzGgoubmV3cy5OZXdzIgASIwoHQWRkTmV3cxIKLm5ld3MuTmV3cxoKLm5ld3MuTmV3cyIASpQDCgYSBAAADgEKCAoBDBIDAAASCggKAQISAwIADQoJCgIDABIDBAAYCgkKAgMBEgMFACUKCgoCBgASBAcADgEKCgoDBgABEgMHCBMKCwoEBgACABIDCAI9CgwKBQYAAgABEgMIBhAKDAoFBgACAAISAwgRJgoMCgUGAAIAAxIDCDE5CgsKBAYAAgESAwkCJwoMCgUGAAIBARIDCQYNCgwKBQYAAgECEgMJDhQKDAoFBgACAQMSAwkfIwoLCgQGAAICEgMKAjsKDAoFBgACAgESAwoGFQoMCgUGAAICAhIDChYkCgwKBQYAAgIDEgMKLzcKCwoEBgACAxIDCwI7CgwKBQYAAgMBEgMLBhAKDAoFBgACAwISAwsRFwoMCgUGAAIDAxIDCyI3CgsKBAYAAgQSAwwCJgoMCgUGAAIEARIDDAYOCgwKBQYAAgQCEgMMDxMKDAoFBgACBAMSAwweIgoLCgQGAAIFEgMNAiUKDAoFBgACBQESAw0GDQoMCgUGAAIFAhIDDQ4SCgwKBQYAAgUDEgMNHSFiBnByb3RvMw== \ No newline at end of file diff --git a/src/core/proto_reader/fixtures/dto_b64.txt b/src/core/proto_reader/fixtures/dto_b64.txt new file mode 100644 index 0000000000..891873f257 --- /dev/null +++ b/src/core/proto_reader/fixtures/dto_b64.txt @@ -0,0 +1 @@ +AAAABHESEBoObmV3c19kdG8ucHJvdG8i3AgK2QgKDm5ld3NfZHRvLnByb3RvEgRuZXdzGhtnb29nbGUvcHJvdG9idWYvZW1wdHkucHJvdG8ihAEKBE5ld3MSDgoCaWQYASABKAVSAmlkEhQKBXRpdGxlGAIgASgJUgV0aXRsZRISCgRib2R5GAMgASgJUgRib2R5EhwKCXBvc3RJbWFnZRgEIAEoCVIJcG9zdEltYWdlEiQKBnN0YXR1cxgFIAEoDjIMLm5ld3MuU3RhdHVzUgZzdGF0dXMiGAoGTmV3c0lkEg4KAmlkGAEgASgFUgJpZCIwCg5NdWx0aXBsZU5ld3NJZBIeCgNpZHMYASADKAsyDC5uZXdzLk5ld3NJZFIDaWRzIioKCE5ld3NMaXN0Eh4KBG5ld3MYASADKAsyCi5uZXdzLk5ld3NSBG5ld3MqLwoGU3RhdHVzEg0KCVBVQkxJU0hFRBAAEgkKBURSQUZUEAESCwoHREVMRVRFRBACSusFCgYSBAAAGCwKCAoBDBIDAAASCggKAQISAwIADQoJCgIDABIDBAAlCgoKAgUAEgQGAAoBCgoKAwUAARIDBgULCgsKBAUAAgASAwcCEAoMCgUFAAIAARIDBwILCgwKBQUAAgACEgMHDg8KCwoEBQACARIDCAIMCgwKBQUAAgEBEgMIAgcKDAoFBQACAQISAwgKCwoLCgQFAAICEgMJAg4KDAoFBQACAgESAwkCCQoMCgUFAAICAhIDCQwNCgoKAgQAEgQMABIBCgoKAwQAARIDDAgMCgsKBAQAAgASAw0CDwoMCgUEAAIABRIDDQIHCgwKBQQAAgABEgMNCAoKDAoFBAACAAMSAw0NDgoLCgQEAAIBEgMOAhMKDAoFBAACAQUSAw4CCAoMCgUEAAIBARIDDgkOCgwKBQQAAgEDEgMOERIKCwoEBAACAhIDDwISCgwKBQQAAgIFEgMPAggKDAoFBAACAgESAw8JDQoMCgUEAAICAxIDDxARCgsKBAQAAgMSAxACFwoMCgUEAAIDBRIDEAIICgwKBQQAAgMBEgMQCRIKDAoFBAACAwMSAxAVFgoLCgQEAAIEEgMRAhQKDAoFBAACBAYSAxECCAoMCgUEAAIEARIDEQkPCgwKBQQAAgQDEgMREhMKCQoCBAESAxQAIAoKCgMEAQESAxQIDgoLCgQEAQIAEgMUER4KDAoFBAECAAUSAxQRFgoMCgUEAQIAARIDFBcZCgwKBQQBAgADEgMUHB0KCQoCBAISAxYAMwoKCgMEAgESAxYIFgoLCgQEAgIAEgMWGTEKDAoFBAICAAQSAxYZIQoMCgUEAgIABhIDFiIoCgwKBQQCAgABEgMWKSwKDAoFBAICAAMSAxYvMAoJCgIEAxIDGAAsCgoKAwQDARIDGAgQCgsKBAQDAgASAxgTKgoMCgUEAwIABBIDGBMbCgwKBQQDAgAGEgMYHCAKDAoFBAMCAAESAxghJQoMCgUEAwIAAxIDGCgpYgZwcm90bzM= \ No newline at end of file diff --git a/tailcall-fixtures/README.md b/tailcall-fixtures/README.md index beb81f5b2b..40e98ee130 100644 --- a/tailcall-fixtures/README.md +++ b/tailcall-fixtures/README.md @@ -1,3 +1,12 @@ # Tailcall fixtures Package that contains configs and fixtures used in tests and examples to be shared among different parts of tailcall plus helper functions to resolve these files in tests. + + +## gRPC binary files + +Instructions to generate those files: +1. go to `src/core/proto_reader/fetch.rs` +2. modify `GrpcReflection.execute` function to print the request `body` as string and response `body` as base64 encoded +3. convert the base64 into bin files using online base64 to file websites or manually +4. rename the files and replace the old ones \ No newline at end of file diff --git a/tailcall-fixtures/fixtures/grpc/reflection/news-list-services.bin b/tailcall-fixtures/fixtures/grpc/reflection/list-services.bin similarity index 100% rename from tailcall-fixtures/fixtures/grpc/reflection/news-list-services.bin rename to tailcall-fixtures/fixtures/grpc/reflection/list-services.bin diff --git a/tailcall-fixtures/fixtures/grpc/reflection/news-service-descriptor.bin b/tailcall-fixtures/fixtures/grpc/reflection/news-service-descriptor.bin deleted file mode 100644 index b1e2f259743ad8c5e3daeba7af265cc63afdfc14..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1373 zcmZuxU2oGs5Ipme5kIz%|2JfFQT7BWX_jYi+Z6I zKaVFDi^avfIx=VH*I$mR%Vm9YXkhV@6nkhQe^!pXU2i|oT?*=1J+DSSV5~c&&KKp) zNE(z;yj-m6v&&Bx)kyJC(Wj6-&dN=f;tuuRUC-;;a$Z?BBRMNqXd;iTpxB{^$+xpr zty41d8r}?b{M(~8(>9@gMD)6N_2fTGiK*7P*JY{xBXos^{+W#jnGj~0bm39;)@M4qpu z5?zYPmqJTl>|vX}5?Y0#?co`$r)3y*s7*=l`fYBKYLgQ#bw@Hq3(W(EK?gq$-2!ER_X1Ho>H%xvdtH5NOK9Zx=v2FA{3 zyNTK{1^}QYeKrgf5bP%7i9?Djn^xdLpl)u=rB=6ifxI8icPmPbITonpRE~`S0H|r! zTssiVr=#XtfM7nAZC45NX(wg#5X|G*)% z>OFa8WhhM4Lrd}1CXv}9O!)mvG83N>WJz()%0Q*+xs~x1rjaZ3ncKeaILOlDr&b1# ztz~>U*?LF`S=KkVgKqpPWP47A>(I*0kC#De+-Z{(yObKAnMIK4+k3VfHlC|}O9hbS e>udpJd2{>d7C@Ho-sNwuqook~JT(7?kN*HlB&XZ} diff --git a/tailcall-fixtures/fixtures/grpc/reflection/news-service.bin b/tailcall-fixtures/fixtures/grpc/reflection/news-service.bin new file mode 100644 index 0000000000000000000000000000000000000000..9964b8ecbfaaa163d1f3cbc526c446d335c7e8ae GIT binary patch literal 808 zcmZ{i-%7(U6vnfp&AQV+n^C7sRP3DIb@&HG#EXFrzb94 z2k#^ul*Oj1{HoQ4YA;Q7oqt2;9}Ku~2#G4-K_w835TR~VFiJwG0cAumhb-!AMKG6i z0Dy8LoT{P#7$-Qddb8kz!wJQ_M~@%?R|&K{-mnUv^=3A#VyLzb42HoP-85E3DIQvd zSS^~f3bER1+hAw`Jv0_Y4EVfN%4@D$g&1`A4Ti#>U<@&TQj#I&&ny^>sbp+1N3ej} w{tjXRk4(@(EU3>~g;=l>n{BROA?+B;MJ$|rn~PW&3F{#iF0b8k{a^2V0p}f$@c;k- literal 0 HcmV?d00001 diff --git a/tailcall-fixtures/fixtures/grpc/reflection/news_dto.bin b/tailcall-fixtures/fixtures/grpc/reflection/news_dto.bin new file mode 100644 index 0000000000000000000000000000000000000000..dc3092d8e3fa45edbe83a43cad77ec53fe463656 GIT binary patch literal 1142 zcmZ9M-EPxB5QW#fYsa40jy*B1`$N@6R7$HVDf9_IEuyF(l$LAMDxr#$;y@gs-t&@3 zya6x4%zC#-z1wfjoa3{06d{EEj@efFZT55ZW!luE<&V0l2Y*8N9m10-YTnw~zN_oI z#cXUgZ`WVPv-@T9V2!^#&>v|FQ;_p%C+yA6G%U- z7tMURnAwe8Q1j`Es;Xhv#kJCeg~1VojFA%+E(>eZ0h24K$(Bd%-kNMV_W47yP>4PLc$!~)5w=(M_~RA%u1O=H5=CA* z;>C?kN0e6VJU3}{;ZmHMG`?UNw%sJBm76b(gMgZroBNKaX*n1=V%y=dBg!tvCnlSY vl{hxp{1&P(Gxw&SBtiekI0z`a>Tf%u>}qGv5!?0;9C6_A@HR028K?gMZ8n2p literal 0 HcmV?d00001 diff --git a/tailcall-fixtures/fixtures/grpc/reflection/protobuf_empty.bin b/tailcall-fixtures/fixtures/grpc/reflection/protobuf_empty.bin new file mode 100644 index 0000000000000000000000000000000000000000..6ce85a30c634012aee539bae2377ab27472d2c68 GIT binary patch literal 2409 zcmbtW&vV;U5LV(+Qc;GExG-gCm(bD{T&L4qnD!*wFU~8DWF$EuoFYrljzHEklAMs~ z^q=9#Kf%D6|Ai|v%rJcW6gO%4bs&f2dH3zN-+sG!Ys<2(rOpR8-=Aqc%hlGR)U}>0 zr(0^isLwj){J@35c@7l&Y;&h|6uU77v z&f{X%(dF!dNqx3Z)z&wKJ}nwni^*?mYu~T82BYo|>$e7mc*h_iapyqg`R9B&;<#sj z+_0}YS8iF`H|$@o+iTaI)-B6<{L^(Z0g2A3yPQr{Srb5?QqS|m~73HXr5iVy$rLWR~ z5J?_qb5%mgzbAoYm$X=sK#l1VQoZD1Ul93UJZ$a*qoz7p&Q(zxiMb_P&|kw-5++5J zS)5lFQkw?BY18H2zAVa9XdogPxQOS9g=vPdnXLlYE1%Ilt1GUkXvRcidgt*OO%zWG zMAW2;6jlJ0Ok%EUOn}j(#&m-nhLfgn*uX_MDm|@Fc}$onCWxwqN_c`0mGNMeJUc}* zMOB3X>0m6PT^Um7??eY~APDDC;P1=6=u`I)PC-3?bQs9NZbZBOurGp;T(1wS7X`9A zj{G3BY10i6x@kPFcSz#PQ6R#Q{D9=%XefaO_`vldDZ(~MuQwd`r8j635Xkc*n}%{v zMz|OGZIkw|apdpNo(OuofN{HWD5FD@a7RWSC)@D@n_L>XK_q+Qp&QU>9E|)>5Lech zVQ=WlJ<;zVKXQ`T7hXi+t~(sQTBuF_fhPiHf4NNR3b1s$L&2$#`VgRj=tW%6#i{r^%0jp>a8vB#_-ML6z;k&MQX zpn>oAjo6_G_GM3mPig40{5v!b1&H+B$Tg{f0lDBu0o`#Zjd0S7L=cQek@UR>=<)%S zLFq1H`$l2k<7#s(NB^)_L^A%XL?CVT;T0~$cqNAM|`5Q6z{||D|joU%SF!1M5Xqt z@fE*@2rtPU$*O}iSk}0illm^6FLG?Hit)_;swu_?yl|!ZcBv|Csq$F2VxR(`G9T$mOZ6n^VSSIjkdfw~?qH>W?H!knL_Fbc6zG^(+-Lx#R z+5W-*w1IiS4;xrM3o*K-TJSDkJ@fb)Kt~4ch%0NfG#B;cYn9Y=^!&vUM#HvSS1fdQ Od*kiNRlK{k-u@E_0o2I= literal 0 HcmV?d00001 diff --git a/tailcall-fixtures/fixtures/protobuf/news.proto b/tailcall-fixtures/fixtures/protobuf/news.proto index 7864a0a53c..05854d8331 100644 --- a/tailcall-fixtures/fixtures/protobuf/news.proto +++ b/tailcall-fixtures/fixtures/protobuf/news.proto @@ -1,22 +1,9 @@ syntax = "proto3"; -import "google/protobuf/empty.proto"; - package news; -enum Status { - PUBLISHED = 0; - DRAFT = 1; - DELETED = 2; -} - -message News { - int32 id = 1; - string title = 2; - string body = 3; - string postImage = 4; - Status status = 5; -} +import "news_dto.proto"; +import "google/protobuf/empty.proto"; service NewsService { rpc GetAllNews(google.protobuf.Empty) returns (NewsList) {} @@ -26,9 +13,3 @@ service NewsService { rpc EditNews(News) returns (News) {} rpc AddNews(News) returns (News) {} } - -message NewsId { int32 id = 1; } - -message MultipleNewsId { repeated NewsId ids = 1; } - -message NewsList { repeated News news = 1; } \ No newline at end of file diff --git a/tailcall-fixtures/fixtures/protobuf/news_dto.proto b/tailcall-fixtures/fixtures/protobuf/news_dto.proto new file mode 100644 index 0000000000..8cec9bc99e --- /dev/null +++ b/tailcall-fixtures/fixtures/protobuf/news_dto.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package news; + +import "google/protobuf/empty.proto"; + +enum Status { + PUBLISHED = 0; + DRAFT = 1; + DELETED = 2; +} + +message News { + int32 id = 1; + string title = 2; + string body = 3; + string postImage = 4; + Status status = 5; +} + +message NewsId { int32 id = 1; } + +message MultipleNewsId { repeated NewsId ids = 1; } + +message NewsList { repeated News news = 1; } diff --git a/tests/core/snapshots/grpc-reflection.md_client.snap b/tests/core/snapshots/grpc-reflection.md_client.snap index 11ebd170a1..c4eb242fdf 100644 --- a/tests/core/snapshots/grpc-reflection.md_client.snap +++ b/tests/core/snapshots/grpc-reflection.md_client.snap @@ -28,6 +28,7 @@ type News { body: String id: Int postImage: String + status: Status title: String } @@ -41,6 +42,12 @@ type Query { news: NewsData! } +enum Status { + DELETED + DRAFT + PUBLISHED +} + scalar UInt128 scalar UInt16 diff --git a/tests/core/snapshots/grpc-reflection.md_merged.snap b/tests/core/snapshots/grpc-reflection.md_merged.snap index 4ca7accb96..3b3bd08d34 100644 --- a/tests/core/snapshots/grpc-reflection.md_merged.snap +++ b/tests/core/snapshots/grpc-reflection.md_merged.snap @@ -9,10 +9,17 @@ schema query: Query } +enum Status { + DELETED + DRAFT + PUBLISHED +} + type News { body: String id: Int postImage: String + status: Status title: String } diff --git a/tests/execution/grpc-reflection.md b/tests/execution/grpc-reflection.md index 134d5e4582..e97daed675 100644 --- a/tests/execution/grpc-reflection.md +++ b/tests/execution/grpc-reflection.md @@ -21,6 +21,13 @@ type News { title: String body: String postImage: String + status: Status +} + +enum Status { + PUBLISHED, + DRAFT, + DELETED, } ``` @@ -31,7 +38,7 @@ type News { textBody: \0\0\0\0\x02:\0 response: status: 200 - fileBody: grpc/reflection/news-list-services.bin + fileBody: grpc/reflection/list-services.bin - request: method: POST @@ -39,7 +46,24 @@ type News { textBody: \0\0\0\0\x12\"\x10news.NewsService response: status: 200 - fileBody: grpc/reflection/news-service-descriptor.bin + fileBody: grpc/reflection/news-service.bin + +- request: + method: POST + url: http://localhost:50051/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo + textBody: \0\0\0\0\x1d\x1a\x1bgoogle/protobuf/empty.proto + expectedHits: 2 + response: + status: 200 + fileBody: grpc/reflection/protobuf_empty.bin + +- request: + method: POST + url: http://localhost:50051/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo + textBody: \0\0\0\0\x10\x1a\x0enews_dto.proto + response: + status: 200 + fileBody: grpc/reflection/news_dto.bin - request: method: POST From 1ece37d0663e7066b44c695011c911f9c5e59a8a Mon Sep 17 00:00:00 2001 From: Panagiotis Date: Wed, 14 Aug 2024 14:42:30 +0300 Subject: [PATCH 3/8] Fix formatting --- src/core/proto_reader/fetch.rs | 5 ++++- tailcall-fixtures/README.md | 4 ++-- tests/execution/grpc-reflection.md | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/core/proto_reader/fetch.rs b/src/core/proto_reader/fetch.rs index 773b21faa2..4c783168af 100644 --- a/src/core/proto_reader/fetch.rs +++ b/src/core/proto_reader/fetch.rs @@ -264,7 +264,10 @@ mod grpc_fetch { let runtime = crate::core::runtime::test::init(None); let resp = grpc_reflection.get_file("news_dto.proto").await?; - let content = runtime.file.read(tailcall_fixtures::protobuf::NEWS_DTO).await?; + let content = runtime + .file + .read(tailcall_fixtures::protobuf::NEWS_DTO) + .await?; let expected = protox_parse::parse("news_dto.proto", &content)?; assert_eq!(expected.name(), resp.name()); diff --git a/tailcall-fixtures/README.md b/tailcall-fixtures/README.md index 40e98ee130..f309efa562 100644 --- a/tailcall-fixtures/README.md +++ b/tailcall-fixtures/README.md @@ -2,11 +2,11 @@ Package that contains configs and fixtures used in tests and examples to be shared among different parts of tailcall plus helper functions to resolve these files in tests. - ## gRPC binary files Instructions to generate those files: + 1. go to `src/core/proto_reader/fetch.rs` 2. modify `GrpcReflection.execute` function to print the request `body` as string and response `body` as base64 encoded 3. convert the base64 into bin files using online base64 to file websites or manually -4. rename the files and replace the old ones \ No newline at end of file +4. rename the files and replace the old ones diff --git a/tests/execution/grpc-reflection.md b/tests/execution/grpc-reflection.md index e97daed675..3abe7739a8 100644 --- a/tests/execution/grpc-reflection.md +++ b/tests/execution/grpc-reflection.md @@ -25,9 +25,9 @@ type News { } enum Status { - PUBLISHED, - DRAFT, - DELETED, + PUBLISHED + DRAFT + DELETED } ``` From 107845f15c7c1f0b8d4acce22d37b29e11103de0 Mon Sep 17 00:00:00 2001 From: Panagiotis Date: Wed, 14 Aug 2024 16:00:17 +0300 Subject: [PATCH 4/8] Fix CF Test --- tailcall-cloudflare/tests/cf_tests.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tailcall-cloudflare/tests/cf_tests.spec.ts b/tailcall-cloudflare/tests/cf_tests.spec.ts index 879f8303f2..fa6812bbfb 100644 --- a/tailcall-cloudflare/tests/cf_tests.spec.ts +++ b/tailcall-cloudflare/tests/cf_tests.spec.ts @@ -8,10 +8,12 @@ describe("fetch", () => { let placeholder_batch = (await readFile("../examples/jsonplaceholder_batch.graphql")).toString() let grpc = (await readFile("../examples/grpc.graphql")).toString() let news_proto = (await readFile("../tailcall-fixtures/fixtures/protobuf/news.proto")).toString() + let news_dto_proto = (await readFile("../tailcall-fixtures/fixtures/protobuf/news_dto.proto")).toString() let bucket = await mf.getR2Bucket("MY_R2") await bucket.put("examples/grpc.graphql", grpc) await bucket.put("examples/../tailcall-fixtures/fixtures/protobuf/news.proto", news_proto) + await bucket.put("examples/../tailcall-fixtures/fixtures/protobuf/news_dto.proto", news_dto_proto) await bucket.put("tailcall-fixtures/fixtures/protobuf/news.proto", grpc) await bucket.put("examples/jsonplaceholder.graphql", placeholder) await bucket.put("examples/jsonplaceholder_batch.graphql", placeholder_batch) From 2bb5e47b9abc50e9ccc99691788ca9c7736e0e23 Mon Sep 17 00:00:00 2001 From: Panagiotis Karatakis Date: Fri, 16 Aug 2024 10:20:26 +0300 Subject: [PATCH 5/8] derive Clone on self instead of cloning manually Co-authored-by: Kiryl Mialeshka <8974488+meskill@users.noreply.github.com> --- src/core/proto_reader/reader.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/core/proto_reader/reader.rs b/src/core/proto_reader/reader.rs index 55f28d072d..4edc7e15e6 100644 --- a/src/core/proto_reader/reader.rs +++ b/src/core/proto_reader/reader.rs @@ -121,14 +121,11 @@ impl ProtoReader { parent_proto: FileDescriptorProto, parent_path: Option<&Path>, ) -> anyhow::Result> { - let parent_path = parent_path.map(|p| p.to_path_buf()); + self.resolve_dependencies(parent_proto, |import| { + let parent_path = parent_path.map(|p| p.to_path_buf()); + let this = self.clone(); - self.resolve_dependencies(parent_proto, move |import| { - let parent_path = parent_path.clone(); - let self_ref = - ProtoReader { reader: self.reader.clone(), runtime: self.runtime.clone() }; - - async move { self_ref.read_proto(import, parent_path.as_deref()).await }.boxed() + async move { this.read_proto(import, parent_path.as_deref()).await }.boxed() }) .await } From 3bd1705c2d98158696c9b766462f270bc6b14f80 Mon Sep 17 00:00:00 2001 From: Panagiotis Karatakis Date: Fri, 16 Aug 2024 10:20:58 +0300 Subject: [PATCH 6/8] Simplify expression Co-authored-by: Kiryl Mialeshka <8974488+meskill@users.noreply.github.com> --- src/core/proto_reader/reader.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/proto_reader/reader.rs b/src/core/proto_reader/reader.rs index 4edc7e15e6..d240266070 100644 --- a/src/core/proto_reader/reader.rs +++ b/src/core/proto_reader/reader.rs @@ -136,8 +136,7 @@ impl ProtoReader { grpc_reflection: Arc, parent_proto: FileDescriptorProto, ) -> anyhow::Result> { - let grpc_reflection = Arc::clone(&grpc_reflection); - self.resolve_dependencies(parent_proto, move |file| { + self.resolve_dependencies(parent_proto, |file| { let grpc_reflection = Arc::clone(&grpc_reflection); async move { grpc_reflection.get_file(file).await }.boxed() }) From 0292fc6d42a12d601c3d88baf891da00a8a8083c Mon Sep 17 00:00:00 2001 From: Panagiotis Date: Fri, 16 Aug 2024 10:30:09 +0300 Subject: [PATCH 7/8] Fix clone issue with lifetime --- src/core/proto_reader/reader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/proto_reader/reader.rs b/src/core/proto_reader/reader.rs index d240266070..47adc31727 100644 --- a/src/core/proto_reader/reader.rs +++ b/src/core/proto_reader/reader.rs @@ -123,7 +123,7 @@ impl ProtoReader { ) -> anyhow::Result> { self.resolve_dependencies(parent_proto, |import| { let parent_path = parent_path.map(|p| p.to_path_buf()); - let this = self.clone(); + let this = ProtoReader { reader: self.reader.clone(), runtime: self.runtime.clone() }; async move { this.read_proto(import, parent_path.as_deref()).await }.boxed() }) From b7df66c426189311bbebfd5cb76d2143f44cad9d Mon Sep 17 00:00:00 2001 From: Panagiotis Date: Fri, 16 Aug 2024 12:40:48 +0300 Subject: [PATCH 8/8] Fix cloning --- src/core/proto_reader/reader.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/proto_reader/reader.rs b/src/core/proto_reader/reader.rs index 47adc31727..1630be0ade 100644 --- a/src/core/proto_reader/reader.rs +++ b/src/core/proto_reader/reader.rs @@ -12,6 +12,7 @@ use crate::core::proto_reader::fetch::GrpcReflection; use crate::core::resource_reader::{Cached, ResourceReader}; use crate::core::runtime::TargetRuntime; +#[derive(Clone)] pub struct ProtoReader { reader: ResourceReader, runtime: TargetRuntime, @@ -123,7 +124,7 @@ impl ProtoReader { ) -> anyhow::Result> { self.resolve_dependencies(parent_proto, |import| { let parent_path = parent_path.map(|p| p.to_path_buf()); - let this = ProtoReader { reader: self.reader.clone(), runtime: self.runtime.clone() }; + let this = self.clone(); async move { this.read_proto(import, parent_path.as_deref()).await }.boxed() })