diff --git a/Cargo.lock b/Cargo.lock index 4e4590344..2761a0857 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -760,7 +760,7 @@ name = "cairo-lang-macro" version = "0.1.0" dependencies = [ "cairo-lang-macro-attributes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cairo-lang-macro-stable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cairo-lang-macro-stable", "linkme", "serde", "serde_json", @@ -790,12 +790,6 @@ dependencies = [ name = "cairo-lang-macro-stable" version = "1.0.0" -[[package]] -name = "cairo-lang-macro-stable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c49906d6b1c215e5814be7c5c65ecf2328898b335bee8c2409ec07cfb5530daf" - [[package]] name = "cairo-lang-parser" version = "2.8.4" @@ -4797,7 +4791,7 @@ dependencies = [ "cairo-lang-formatter", "cairo-lang-lowering", "cairo-lang-macro", - "cairo-lang-macro-stable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cairo-lang-macro-stable", "cairo-lang-parser", "cairo-lang-semantic", "cairo-lang-sierra", diff --git a/plugins/cairo-lang-macro-stable/src/lib.rs b/plugins/cairo-lang-macro-stable/src/lib.rs index dd91b21a8..fc5542987 100644 --- a/plugins/cairo-lang-macro-stable/src/lib.rs +++ b/plugins/cairo-lang-macro-stable/src/lib.rs @@ -1,11 +1,30 @@ use crate::ffi::{StableOption, StableSlice}; -use std::ffi::CStr; use std::num::NonZeroU8; use std::os::raw::c_char; use std::ptr::NonNull; pub mod ffi; +#[repr(C)] +#[derive(Debug)] +pub struct StableToken { + pub span: StableTextSpan, + pub content: *mut c_char, +} + +#[repr(C)] +#[derive(Debug)] +pub struct StableTextSpan { + pub start: usize, + pub end: usize, +} + +#[repr(C)] +#[derive(Debug)] +pub enum StableTokenTree { + Ident(StableToken), +} + #[repr(C)] #[derive(Debug)] pub struct StableExpansion { @@ -23,7 +42,7 @@ pub type StableExpansionsList = StableSlice; #[repr(C)] #[derive(Debug)] pub struct StableTokenStream { - pub value: *mut c_char, + pub tokens: StableSlice, pub metadata: StableTokenStreamMetadata, } @@ -76,17 +95,6 @@ pub struct StableResultWrapper { pub output: StableProcMacroResult, } -impl StableTokenStream { - /// Convert to String. - /// - /// # Safety - pub unsafe fn to_string(&self) -> String { - // Note that this does not deallocate the c-string. - // The memory must still be freed with `CString::from_raw`. - CStr::from_ptr(self.value).to_string_lossy().to_string() - } -} - #[repr(C)] pub struct StablePostProcessContext { pub aux_data: StableSlice, diff --git a/plugins/cairo-lang-macro/Cargo.toml b/plugins/cairo-lang-macro/Cargo.toml index f81d7d091..384a3aa7e 100644 --- a/plugins/cairo-lang-macro/Cargo.toml +++ b/plugins/cairo-lang-macro/Cargo.toml @@ -14,7 +14,7 @@ repository.workspace = true [dependencies] cairo-lang-macro-attributes = "0.1" -cairo-lang-macro-stable = "1" +cairo-lang-macro-stable = { path = "../cairo-lang-macro-stable" } linkme.workspace = true serde = { workspace = true, optional = true } diff --git a/plugins/cairo-lang-macro/src/types/conversion.rs b/plugins/cairo-lang-macro/src/types/conversion.rs index 2e5516d0a..3242afd2f 100644 --- a/plugins/cairo-lang-macro/src/types/conversion.rs +++ b/plugins/cairo-lang-macro/src/types/conversion.rs @@ -1,12 +1,12 @@ use crate::{ AuxData, Diagnostic, ExpansionDefinition, FullPathMarker, PostProcessContext, ProcMacroResult, - Severity, TokenStream, TokenStreamMetadata, + Severity, TextSpan, Token, TokenStream, TokenStreamMetadata, TokenTree, }; use cairo_lang_macro_stable::ffi::StableSlice; use cairo_lang_macro_stable::{ StableAuxData, StableDiagnostic, StableExpansion, StableFullPathMarker, - StablePostProcessContext, StableProcMacroResult, StableSeverity, StableTokenStream, - StableTokenStreamMetadata, + StablePostProcessContext, StableProcMacroResult, StableSeverity, StableTextSpan, StableToken, + StableTokenStream, StableTokenStreamMetadata, StableTokenTree, }; use std::ffi::{c_char, CStr, CString}; use std::num::NonZeroU8; @@ -90,15 +90,120 @@ impl ProcMacroResult { } } +impl TextSpan { + /// Convert to FFI-safe representation. + #[doc(hidden)] + pub fn into_stable(self) -> StableTextSpan { + StableTextSpan { + start: self.start, + end: self.end, + } + } + + #[doc(hidden)] + pub fn from_stable(span: &StableTextSpan) -> Self { + Self { + start: span.start, + end: span.end, + } + } + + #[doc(hidden)] + pub fn from_owned_stable(span: StableTextSpan) -> Self { + Self { + start: span.start, + end: span.end, + } + } +} + +impl Token { + /// Convert to FFI-safe representation. + #[doc(hidden)] + pub fn into_stable(self) -> StableToken { + let cstr = CString::new(self.content.as_bytes()).unwrap(); + StableToken { + span: self.span.into_stable(), + content: cstr.into_raw(), + } + } + + /// Convert to native Rust representation, without taking the ownership of the string. + /// + /// Note that you still need to free the memory by calling `from_owned_stable`. + /// + /// # Safety + #[doc(hidden)] + pub unsafe fn from_stable(token: &StableToken) -> Self { + Self { + content: from_raw_cstr(token.content), + span: TextSpan::from_stable(&token.span), + } + } + + /// Convert to native Rust representation, with taking the ownership of the string. + /// + /// Useful when you need to free the allocated memory. + /// Only use on the same side of FFI-barrier, where the memory has been allocated. + /// + /// # Safety + #[doc(hidden)] + pub unsafe fn from_owned_stable(token: StableToken) -> Self { + Self { + content: from_raw_cstring(token.content), + span: TextSpan::from_owned_stable(token.span), + } + } +} + +impl TokenTree { + /// Convert to FFI-safe representation. + #[doc(hidden)] + pub fn into_stable(self) -> StableTokenTree { + match self { + Self::Ident(token) => StableTokenTree::Ident(token.into_stable()), + } + } + + /// Convert to native Rust representation, without taking the ownership of the string. + /// + /// Note that you still need to free the memory by calling `from_owned_stable`. + /// + /// # Safety + #[doc(hidden)] + pub unsafe fn from_stable(token_tree: &StableTokenTree) -> Self { + match token_tree { + StableTokenTree::Ident(token) => Self::Ident(Token::from_stable(token)), + } + } + + /// Convert to native Rust representation, with taking the ownership of the string. + /// + /// Useful when you need to free the allocated memory. + /// Only use on the same side of FFI-barrier, where the memory has been allocated. + /// + /// # Safety + #[doc(hidden)] + pub unsafe fn from_owned_stable(token_tree: StableTokenTree) -> Self { + match token_tree { + StableTokenTree::Ident(token) => Self::Ident(Token::from_owned_stable(token)), + } + } +} + impl TokenStream { /// Convert to FFI-safe representation. /// /// # Safety #[doc(hidden)] pub fn into_stable(self) -> StableTokenStream { - let cstr = CString::new(self.value).unwrap(); + let tokens = self + .tokens + .into_iter() + .map(|token| token.into_stable()) + .collect::>(); StableTokenStream { - value: cstr.into_raw(), + tokens: StableSlice::new(tokens), metadata: self.metadata.into_stable(), } } @@ -110,8 +215,13 @@ impl TokenStream { /// # Safety #[doc(hidden)] pub unsafe fn from_stable(token_stream: &StableTokenStream) -> Self { + let (ptr, n) = token_stream.tokens.raw_parts(); + let tokens = slice::from_raw_parts(ptr, n) + .iter() + .map(|token_tree| TokenTree::from_stable(token_tree)) + .collect::>(); Self { - value: from_raw_cstr(token_stream.value), + tokens, metadata: TokenStreamMetadata::from_stable(&token_stream.metadata), } } @@ -124,8 +234,13 @@ impl TokenStream { /// # Safety #[doc(hidden)] pub unsafe fn from_owned_stable(token_stream: StableTokenStream) -> Self { + let tokens = token_stream.tokens.into_owned(); + let tokens = tokens + .into_iter() + .map(|token_tree| TokenTree::from_owned_stable(token_tree)) + .collect::>(); Self { - value: from_raw_cstring(token_stream.value), + tokens, metadata: TokenStreamMetadata::from_owned_stable(token_stream.metadata), } } diff --git a/plugins/cairo-lang-macro/src/types/mod.rs b/plugins/cairo-lang-macro/src/types/mod.rs index a811d7558..618d61746 100644 --- a/plugins/cairo-lang-macro/src/types/mod.rs +++ b/plugins/cairo-lang-macro/src/types/mod.rs @@ -1,10 +1,11 @@ -use std::fmt::Display; use std::vec::IntoIter; mod conversion; mod expansions; +mod token; pub use expansions::*; +pub use token::*; /// Result of procedural macro code generation. #[derive(Debug, Clone)] @@ -15,78 +16,6 @@ pub struct ProcMacroResult { pub full_path_markers: Vec, } -/// An abstract stream of Cairo tokens. -/// -/// This is both input and part of an output of a procedural macro. -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] -pub struct TokenStream { - value: String, - metadata: TokenStreamMetadata, -} - -/// Metadata of [`TokenStream`]. -/// -/// This struct can be used to describe the origin of the [`TokenStream`]. -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] -pub struct TokenStreamMetadata { - /// The path to the file from which the [`TokenStream`] has been created. - pub original_file_path: Option, - /// ID of the file from which the [`TokenStream`] has been created. - /// - /// It is guaranteed, that the `file_id` will be unique for each file. - pub file_id: Option, -} - -impl TokenStream { - #[doc(hidden)] - pub fn new(value: String) -> Self { - Self { - value, - metadata: TokenStreamMetadata::default(), - } - } - - #[doc(hidden)] - pub fn empty() -> Self { - Self::new("".to_string()) - } - - #[doc(hidden)] - pub fn with_metadata(mut self, metadata: TokenStreamMetadata) -> Self { - self.metadata = metadata; - self - } - - /// Get `[TokenStreamMetadata`] associated with this [`TokenStream`]. - /// - /// The metadata struct can be used to describe the [`TokenStream`] origin. - pub fn metadata(&self) -> &TokenStreamMetadata { - &self.metadata - } - - pub fn is_empty(&self) -> bool { - self.to_string().is_empty() - } -} - -impl Display for TokenStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.value) - } -} - -impl TokenStreamMetadata { - #[doc(hidden)] - pub fn new(file_path: impl ToString, file_id: impl ToString) -> Self { - Self { - original_file_path: Some(file_path.to_string()), - file_id: Some(file_id.to_string()), - } - } -} - /// **Auxiliary data** returned by procedural macro code generation. /// /// This struct can be used to collect additional information from the Cairo source code of @@ -101,7 +30,7 @@ impl TokenStreamMetadata { /// For instance, auxiliary data can be serialized as JSON. /// /// ``` -/// use cairo_lang_macro::{AuxData, ProcMacroResult, TokenStream, attribute_macro, post_process, PostProcessContext}; +/// use cairo_lang_macro::{AuxData, ProcMacroResult, TokenStream, TokenTree, Token, TextSpan, attribute_macro, post_process, PostProcessContext}; /// use serde::{Serialize, Deserialize}; /// #[derive(Debug, Serialize, Deserialize)] /// struct SomeAuxDataFormat { @@ -110,11 +39,16 @@ impl TokenStreamMetadata { /// /// #[attribute_macro] /// pub fn some_macro(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { -/// let token_stream = TokenStream::new( -/// token_stream.to_string() -/// // Remove macro call to avoid infinite loop. -/// .replace("#[some]", "") -/// ); +/// // Remove macro call to avoid infinite loop. +/// let code = token_stream.to_string().replace("#[some]", ""); +/// let token_stream = TokenStream::new(vec![ +/// TokenTree::Ident( +/// Token::new( +/// code.clone(), +/// TextSpan::new(0, code.len()) +/// ) +/// ) +/// ]); /// let value = SomeAuxDataFormat { some_message: "Hello from some macro!".to_string() }; /// let value = serde_json::to_string(&value).unwrap(); /// let value: Vec = value.into_bytes(); diff --git a/plugins/cairo-lang-macro/src/types/token.rs b/plugins/cairo-lang-macro/src/types/token.rs new file mode 100644 index 000000000..30e289982 --- /dev/null +++ b/plugins/cairo-lang-macro/src/types/token.rs @@ -0,0 +1,129 @@ +use std::fmt::Display; + +/// An abstract stream of Cairo tokens. +/// +/// This is both input and part of an output of a procedural macro. +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub struct TokenStream { + pub tokens: Vec, + pub metadata: TokenStreamMetadata, +} + +/// A single token or a delimited sequence of token trees. +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum TokenTree { + Ident(Token), +} + +impl Default for TokenTree { + fn default() -> Self { + Self::Ident(Default::default()) + } +} + +/// A range of text offsets that form a span (like text selection). +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub struct TextSpan { + pub start: usize, + pub end: usize, +} + +/// A single Cairo token. +/// +/// The most atomic item, of Cairo code representation, when passed between macro and host. +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub struct Token { + pub content: String, + pub span: TextSpan, +} + +/// Metadata of [`TokenStream`]. +/// +/// This struct can be used to describe the origin of the [`TokenStream`]. +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub struct TokenStreamMetadata { + /// The path to the file from which the [`TokenStream`] has been created. + pub original_file_path: Option, + /// ID of the file from which the [`TokenStream`] has been created. + /// + /// It is guaranteed, that the `file_id` will be unique for each file. + pub file_id: Option, +} + +impl TokenStream { + #[doc(hidden)] + pub fn new(tokens: Vec) -> Self { + Self { + tokens, + metadata: TokenStreamMetadata::default(), + } + } + + #[doc(hidden)] + pub fn empty() -> Self { + Self::new(Vec::default()) + } + + #[doc(hidden)] + pub fn with_metadata(mut self, metadata: TokenStreamMetadata) -> Self { + self.metadata = metadata; + self + } + + /// Get `[TokenStreamMetadata`] associated with this [`TokenStream`]. + /// + /// The metadata struct can be used to describe the [`TokenStream`] origin. + pub fn metadata(&self) -> &TokenStreamMetadata { + &self.metadata + } + + pub fn is_empty(&self) -> bool { + self.to_string().is_empty() + } +} + +impl Display for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for token in &self.tokens { + match token { + TokenTree::Ident(token) => { + write!(f, "{}", token.content.clone())?; + } + } + } + Ok(()) + } +} + +impl TokenStreamMetadata { + #[doc(hidden)] + pub fn new(file_path: impl ToString, file_id: impl ToString) -> Self { + Self { + original_file_path: Some(file_path.to_string()), + file_id: Some(file_id.to_string()), + } + } +} + +impl TokenTree { + pub fn from_ident(token: Token) -> Self { + Self::Ident(token) + } +} + +impl TextSpan { + pub fn new(start: usize, end: usize) -> TextSpan { + TextSpan { start, end } + } +} + +impl Token { + pub fn new(content: String, span: TextSpan) -> Self { + Self { content, span } + } +} diff --git a/scarb/Cargo.toml b/scarb/Cargo.toml index 8693d8725..6ccc50698 100644 --- a/scarb/Cargo.toml +++ b/scarb/Cargo.toml @@ -23,7 +23,7 @@ cairo-lang-filesystem.workspace = true cairo-lang-formatter.workspace = true cairo-lang-lowering.workspace = true cairo-lang-macro = { path = "../plugins/cairo-lang-macro" } -cairo-lang-macro-stable = "1" +cairo-lang-macro-stable = { path = "../plugins/cairo-lang-macro-stable" } cairo-lang-parser.workspace = true cairo-lang-semantic.workspace = true cairo-lang-sierra-to-casm.workspace = true diff --git a/scarb/src/compiler/plugin/proc_macro/ffi.rs b/scarb/src/compiler/plugin/proc_macro/ffi.rs index 4d42bab6c..a075e5d8f 100644 --- a/scarb/src/compiler/plugin/proc_macro/ffi.rs +++ b/scarb/src/compiler/plugin/proc_macro/ffi.rs @@ -1,6 +1,5 @@ use crate::core::{Config, Package, PackageId}; use anyhow::{ensure, Context, Result}; -use cairo_lang_defs::patcher::PatchBuilder; use cairo_lang_macro::{ ExpansionKind as SharedExpansionKind, FullPathMarker, PostProcessContext, ProcMacroResult, TokenStream, @@ -9,8 +8,6 @@ use cairo_lang_macro_stable::{ StableExpansion, StableExpansionsList, StablePostProcessContext, StableProcMacroResult, StableResultWrapper, StableTokenStream, }; -use cairo_lang_syntax::node::db::SyntaxGroup; -use cairo_lang_syntax::node::TypedSyntaxNode; use camino::Utf8PathBuf; use itertools::Itertools; use libloading::{Library, Symbol}; @@ -27,18 +24,6 @@ use libloading::os::unix::Symbol as RawSymbol; use libloading::os::windows::Symbol as RawSymbol; use smol_str::SmolStr; -pub trait FromSyntaxNode { - fn from_syntax_node(db: &dyn SyntaxGroup, node: &impl TypedSyntaxNode) -> Self; -} - -impl FromSyntaxNode for TokenStream { - fn from_syntax_node(db: &dyn SyntaxGroup, node: &impl TypedSyntaxNode) -> Self { - let mut builder = PatchBuilder::new(db, node); - builder.add_node(node.as_syntax_node()); - Self::new(builder.build().0) - } -} - const EXEC_ATTR_PREFIX: &str = "__exec_attr_"; /// Representation of a single procedural macro. diff --git a/scarb/src/compiler/plugin/proc_macro/host.rs b/scarb/src/compiler/plugin/proc_macro/host.rs index 32694ca89..c48a8931f 100644 --- a/scarb/src/compiler/plugin/proc_macro/host.rs +++ b/scarb/src/compiler/plugin/proc_macro/host.rs @@ -1,5 +1,5 @@ use crate::compiler::plugin::proc_macro::{ - Expansion, ExpansionKind, FromSyntaxNode, ProcMacroInstance, + Expansion, ExpansionKind, ProcMacroInstance, TokenStreamBuilder, }; use crate::core::{Config, Package, PackageId}; use anyhow::{ensure, Result}; @@ -188,12 +188,12 @@ impl ProcMacroHostPlugin { continue; }; - let mut func_builder = PatchBuilder::new(db, func); + let mut token_stream_builder = TokenStreamBuilder::new(db); let attrs = func.attributes(db).elements(db); - let found = self.parse_attrs(db, &mut func_builder, attrs, func); - func_builder.add_node(func.declaration(db).as_syntax_node()); - func_builder.add_node(func.body(db).as_syntax_node()); - let token_stream = TokenStream::new(func_builder.build().0); + let found = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(func.declaration(db).as_syntax_node()); + token_stream_builder.add_node(func.body(db).as_syntax_node()); + let token_stream = token_stream_builder.build(); all_none = all_none && self.do_expand_inner_attr( @@ -242,13 +242,13 @@ impl ProcMacroHostPlugin { continue; }; - let mut func_builder = PatchBuilder::new(db, &func); + let mut token_stream_builder = TokenStreamBuilder::new(db); let attrs = func.attributes(db).elements(db); - let found = self.parse_attrs(db, &mut func_builder, attrs, &func); - func_builder.add_node(func.visibility(db).as_syntax_node()); - func_builder.add_node(func.declaration(db).as_syntax_node()); - func_builder.add_node(func.body(db).as_syntax_node()); - let token_stream = TokenStream::new(func_builder.build().0); + let found = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(func.visibility(db).as_syntax_node()); + token_stream_builder.add_node(func.declaration(db).as_syntax_node()); + token_stream_builder.add_node(func.body(db).as_syntax_node()); + let token_stream = token_stream_builder.build(); all_none = all_none && self.do_expand_inner_attr( db, @@ -331,103 +331,102 @@ impl ProcMacroHostPlugin { db: &dyn SyntaxGroup, item_ast: ast::ModuleItem, ) -> (AttrExpansionFound, TokenStream) { - let mut item_builder = PatchBuilder::new(db, &item_ast); + let mut token_stream_builder = TokenStreamBuilder::new(db); let input = match item_ast.clone() { ast::ModuleItem::Trait(trait_ast) => { let attrs = trait_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut item_builder, attrs, &item_ast); - item_builder.add_node(trait_ast.visibility(db).as_syntax_node()); - item_builder.add_node(trait_ast.trait_kw(db).as_syntax_node()); - item_builder.add_node(trait_ast.name(db).as_syntax_node()); - item_builder.add_node(trait_ast.generic_params(db).as_syntax_node()); - item_builder.add_node(trait_ast.body(db).as_syntax_node()); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(trait_ast.visibility(db).as_syntax_node()); + token_stream_builder.add_node(trait_ast.trait_kw(db).as_syntax_node()); + token_stream_builder.add_node(trait_ast.name(db).as_syntax_node()); + token_stream_builder.add_node(trait_ast.generic_params(db).as_syntax_node()); + token_stream_builder.add_node(trait_ast.body(db).as_syntax_node()); expansion } ast::ModuleItem::Impl(impl_ast) => { let attrs = impl_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut item_builder, attrs, &item_ast); - item_builder.add_node(impl_ast.visibility(db).as_syntax_node()); - item_builder.add_node(impl_ast.impl_kw(db).as_syntax_node()); - item_builder.add_node(impl_ast.name(db).as_syntax_node()); - item_builder.add_node(impl_ast.generic_params(db).as_syntax_node()); - item_builder.add_node(impl_ast.of_kw(db).as_syntax_node()); - item_builder.add_node(impl_ast.trait_path(db).as_syntax_node()); - item_builder.add_node(impl_ast.body(db).as_syntax_node()); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(impl_ast.visibility(db).as_syntax_node()); + token_stream_builder.add_node(impl_ast.impl_kw(db).as_syntax_node()); + token_stream_builder.add_node(impl_ast.name(db).as_syntax_node()); + token_stream_builder.add_node(impl_ast.generic_params(db).as_syntax_node()); + token_stream_builder.add_node(impl_ast.of_kw(db).as_syntax_node()); + token_stream_builder.add_node(impl_ast.trait_path(db).as_syntax_node()); + token_stream_builder.add_node(impl_ast.body(db).as_syntax_node()); expansion } ast::ModuleItem::Module(module_ast) => { let attrs = module_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut item_builder, attrs, &item_ast); - item_builder.add_node(module_ast.visibility(db).as_syntax_node()); - item_builder.add_node(module_ast.module_kw(db).as_syntax_node()); - item_builder.add_node(module_ast.name(db).as_syntax_node()); - item_builder.add_node(module_ast.body(db).as_syntax_node()); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(module_ast.visibility(db).as_syntax_node()); + token_stream_builder.add_node(module_ast.module_kw(db).as_syntax_node()); + token_stream_builder.add_node(module_ast.name(db).as_syntax_node()); + token_stream_builder.add_node(module_ast.body(db).as_syntax_node()); expansion } ast::ModuleItem::FreeFunction(free_func_ast) => { let attrs = free_func_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut item_builder, attrs, &item_ast); - item_builder.add_node(free_func_ast.visibility(db).as_syntax_node()); - item_builder.add_node(free_func_ast.declaration(db).as_syntax_node()); - item_builder.add_node(free_func_ast.body(db).as_syntax_node()); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(free_func_ast.visibility(db).as_syntax_node()); + token_stream_builder.add_node(free_func_ast.declaration(db).as_syntax_node()); + token_stream_builder.add_node(free_func_ast.body(db).as_syntax_node()); expansion } ast::ModuleItem::ExternFunction(extern_func_ast) => { let attrs = extern_func_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut item_builder, attrs, &item_ast); - item_builder.add_node(extern_func_ast.visibility(db).as_syntax_node()); - item_builder.add_node(extern_func_ast.extern_kw(db).as_syntax_node()); - item_builder.add_node(extern_func_ast.declaration(db).as_syntax_node()); - item_builder.add_node(extern_func_ast.semicolon(db).as_syntax_node()); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(extern_func_ast.visibility(db).as_syntax_node()); + token_stream_builder.add_node(extern_func_ast.extern_kw(db).as_syntax_node()); + token_stream_builder.add_node(extern_func_ast.declaration(db).as_syntax_node()); + token_stream_builder.add_node(extern_func_ast.semicolon(db).as_syntax_node()); expansion } ast::ModuleItem::ExternType(extern_type_ast) => { let attrs = extern_type_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut item_builder, attrs, &item_ast); - item_builder.add_node(extern_type_ast.visibility(db).as_syntax_node()); - item_builder.add_node(extern_type_ast.extern_kw(db).as_syntax_node()); - item_builder.add_node(extern_type_ast.type_kw(db).as_syntax_node()); - item_builder.add_node(extern_type_ast.name(db).as_syntax_node()); - item_builder.add_node(extern_type_ast.generic_params(db).as_syntax_node()); - item_builder.add_node(extern_type_ast.semicolon(db).as_syntax_node()); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(extern_type_ast.visibility(db).as_syntax_node()); + token_stream_builder.add_node(extern_type_ast.extern_kw(db).as_syntax_node()); + token_stream_builder.add_node(extern_type_ast.type_kw(db).as_syntax_node()); + token_stream_builder.add_node(extern_type_ast.name(db).as_syntax_node()); + token_stream_builder.add_node(extern_type_ast.generic_params(db).as_syntax_node()); + token_stream_builder.add_node(extern_type_ast.semicolon(db).as_syntax_node()); expansion } ast::ModuleItem::Struct(struct_ast) => { let attrs = struct_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut item_builder, attrs, &item_ast); - item_builder.add_node(struct_ast.visibility(db).as_syntax_node()); - item_builder.add_node(struct_ast.struct_kw(db).as_syntax_node()); - item_builder.add_node(struct_ast.name(db).as_syntax_node()); - item_builder.add_node(struct_ast.generic_params(db).as_syntax_node()); - item_builder.add_node(struct_ast.lbrace(db).as_syntax_node()); - item_builder.add_node(struct_ast.members(db).as_syntax_node()); - item_builder.add_node(struct_ast.rbrace(db).as_syntax_node()); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(struct_ast.visibility(db).as_syntax_node()); + token_stream_builder.add_node(struct_ast.struct_kw(db).as_syntax_node()); + token_stream_builder.add_node(struct_ast.name(db).as_syntax_node()); + token_stream_builder.add_node(struct_ast.generic_params(db).as_syntax_node()); + token_stream_builder.add_node(struct_ast.lbrace(db).as_syntax_node()); + token_stream_builder.add_node(struct_ast.members(db).as_syntax_node()); + token_stream_builder.add_node(struct_ast.rbrace(db).as_syntax_node()); expansion } ast::ModuleItem::Enum(enum_ast) => { let attrs = enum_ast.attributes(db).elements(db); - let expansion = self.parse_attrs(db, &mut item_builder, attrs, &item_ast); - item_builder.add_node(enum_ast.visibility(db).as_syntax_node()); - item_builder.add_node(enum_ast.enum_kw(db).as_syntax_node()); - item_builder.add_node(enum_ast.name(db).as_syntax_node()); - item_builder.add_node(enum_ast.generic_params(db).as_syntax_node()); - item_builder.add_node(enum_ast.lbrace(db).as_syntax_node()); - item_builder.add_node(enum_ast.variants(db).as_syntax_node()); - item_builder.add_node(enum_ast.rbrace(db).as_syntax_node()); + let expansion = self.parse_attrs(db, &mut token_stream_builder, attrs); + token_stream_builder.add_node(enum_ast.visibility(db).as_syntax_node()); + token_stream_builder.add_node(enum_ast.enum_kw(db).as_syntax_node()); + token_stream_builder.add_node(enum_ast.name(db).as_syntax_node()); + token_stream_builder.add_node(enum_ast.generic_params(db).as_syntax_node()); + token_stream_builder.add_node(enum_ast.lbrace(db).as_syntax_node()); + token_stream_builder.add_node(enum_ast.variants(db).as_syntax_node()); + token_stream_builder.add_node(enum_ast.rbrace(db).as_syntax_node()); expansion } _ => AttrExpansionFound::None, }; - let token_stream = TokenStream::new(item_builder.build().0); + let token_stream = token_stream_builder.build(); (input, token_stream) } fn parse_attrs( &self, db: &dyn SyntaxGroup, - builder: &mut PatchBuilder<'_>, + builder: &mut TokenStreamBuilder<'_>, attrs: Vec, - origin: &impl TypedSyntaxNode, ) -> AttrExpansionFound { // This function parses attributes of the item, // checking if those attributes correspond to a procedural macro that should be fired. @@ -450,9 +449,9 @@ impl ProcMacroHostPlugin { )); if let Some(found) = found { if expansion.is_none() { - let mut args_builder = PatchBuilder::new(db, origin); + let mut args_builder = TokenStreamBuilder::new(db); args_builder.add_node(attr.arguments(db).as_syntax_node()); - let args = TokenStream::new(args_builder.build().0); + let args = args_builder.build(); expansion = Some((found, args, attr.stable_ptr().untyped())); // Do not add the attribute for found expansion. continue; @@ -523,9 +522,10 @@ impl ProcMacroHostPlugin { stream_metadata: TokenStreamMetadata, ) -> Option { let stable_ptr = item_ast.clone().stable_ptr().untyped(); - let token_stream = - TokenStream::from_syntax_node(db, &item_ast).with_metadata(stream_metadata.clone()); - + let mut token_stream_builder = TokenStreamBuilder::new(db); + token_stream_builder.add_node(item_ast.as_syntax_node()); + token_stream_builder.with_metadata(stream_metadata.clone()); + let token_stream = token_stream_builder.build(); let mut aux_data = EmittedAuxData::default(); let mut all_diagnostics: Vec = Vec::new(); @@ -1019,7 +1019,9 @@ impl InlineMacroExprPlugin for ProcMacroInlinePlugin { _metadata: &MacroPluginMetadata<'_>, ) -> InlinePluginResult { let stable_ptr = syntax.clone().stable_ptr().untyped(); - let token_stream = TokenStream::from_syntax_node(db, syntax); + let mut token_stream_builder = TokenStreamBuilder::new(db); + token_stream_builder.add_node(syntax.as_syntax_node()); + let token_stream = token_stream_builder.build(); let result = self.instance().generate_code( self.expansion.name.clone(), TokenStream::empty(), diff --git a/scarb/src/compiler/plugin/proc_macro/mod.rs b/scarb/src/compiler/plugin/proc_macro/mod.rs index 888c012fc..83a4e7822 100644 --- a/scarb/src/compiler/plugin/proc_macro/mod.rs +++ b/scarb/src/compiler/plugin/proc_macro/mod.rs @@ -1,7 +1,9 @@ pub mod compilation; mod ffi; mod host; +mod types; pub use compilation::{check_unit, compile_unit, fetch_crate}; pub use ffi::*; pub use host::*; +pub use types::*; diff --git a/scarb/src/compiler/plugin/proc_macro/types.rs b/scarb/src/compiler/plugin/proc_macro/types.rs new file mode 100644 index 000000000..dc8f5439b --- /dev/null +++ b/scarb/src/compiler/plugin/proc_macro/types.rs @@ -0,0 +1,54 @@ +use cairo_lang_macro::{TextSpan, Token, TokenStream, TokenStreamMetadata, TokenTree}; +use cairo_lang_syntax::node::{db::SyntaxGroup, SyntaxNode}; + +/// Helps creating TokenStream based on multiple SyntaxNodes, +/// which aren't descendants or ascendants of each other inside the SyntaxTree. +pub struct TokenStreamBuilder<'a> { + db: &'a dyn SyntaxGroup, + nodes: Vec, + metadata: Option, +} + +impl<'a> TokenStreamBuilder<'a> { + pub fn new(db: &'a dyn SyntaxGroup) -> Self { + Self { + db, + nodes: Vec::default(), + metadata: None, + } + } + + pub fn add_node(&mut self, node: SyntaxNode) { + self.nodes.push(node); + } + + pub fn with_metadata(&mut self, metadata: TokenStreamMetadata) { + self.metadata = Some(metadata); + } + + pub fn build(self) -> TokenStream { + let mut result: Vec = Vec::default(); + for node in self.nodes.iter() { + let leaves = node.tokens(self.db); + let tokens = + leaves.map(|node| TokenTree::Ident(self.token_from_syntax_node(node.clone()))); + result.extend(tokens); + } + + match self.metadata { + Some(metadata) => TokenStream::new(result.clone()).with_metadata(metadata.clone()), + None => TokenStream::new(result.clone()), + } + } + + pub fn token_from_syntax_node(&self, node: SyntaxNode) -> Token { + let span = node.span(self.db).to_str_range(); + Token::new( + node.get_text(self.db), + TextSpan { + start: span.start, + end: span.end, + }, + ) + } +} diff --git a/scarb/src/ops/proc_macro_server/methods/expand_derive.rs b/scarb/src/ops/proc_macro_server/methods/expand_derive.rs index d792ef51e..fc42b846c 100644 --- a/scarb/src/ops/proc_macro_server/methods/expand_derive.rs +++ b/scarb/src/ops/proc_macro_server/methods/expand_derive.rs @@ -10,7 +10,7 @@ use crate::compiler::plugin::proc_macro::{Expansion, ExpansionKind, ProcMacroHos impl Handler for ExpandDerive { fn handle(proc_macro_host: Arc, params: Self::Params) -> Result { - let mut derived_code = String::new(); + let mut derived_code = TokenStream::empty(); let mut all_diagnostics = vec![]; for derive in params.derives { @@ -30,11 +30,11 @@ impl Handler for ExpandDerive { // Register diagnostics. all_diagnostics.extend(result.diagnostics); // Add generated code. - derived_code.push_str(&result.token_stream.to_string()); + derived_code.tokens.extend(result.token_stream.tokens); } Ok(ProcMacroResult { - token_stream: TokenStream::new(derived_code), + token_stream: derived_code, diagnostics: all_diagnostics, }) } diff --git a/scarb/tests/build_cairo_plugin.rs b/scarb/tests/build_cairo_plugin.rs index e9be7668a..2598ced72 100644 --- a/scarb/tests/build_cairo_plugin.rs +++ b/scarb/tests/build_cairo_plugin.rs @@ -372,15 +372,15 @@ fn can_replace_original_node() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro}; + use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, TokenTree, Token, TextSpan}; #[attribute_macro] pub fn some(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - let token_stream = TokenStream::new( - token_stream - .to_string() - .replace("12", "34") - ); + let new_token_string = token_stream.to_string().replace("12", "34"); + let token_stream = TokenStream::new(vec![TokenTree::Ident(Token::new( + new_token_string.clone(), + TextSpan { start: 0, end: new_token_string.len() }, + ))]); ProcMacroResult::new(token_stream) } "##}) @@ -538,26 +538,26 @@ fn can_define_multiple_macros() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, AuxData, PostProcessContext, post_process}; + use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, AuxData, PostProcessContext, post_process, TokenTree, Token, TextSpan}; #[attribute_macro] pub fn hello(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - let token_stream = TokenStream::new( - token_stream - .to_string() - .replace("12", "34") - ); + let new_token_string = token_stream.to_string().replace("12", "34"); + let token_stream = TokenStream::new(vec![TokenTree::Ident(Token::new( + new_token_string.clone(), + TextSpan { start: 0, end: new_token_string.len() }, + ))]); let aux_data = AuxData::new(Vec::new()); ProcMacroResult::new(token_stream).with_aux_data(aux_data) } #[attribute_macro] pub fn world(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - let token_stream = TokenStream::new( - token_stream - .to_string() - .replace("56", "78") - ); + let new_token_string = token_stream.to_string().replace("56", "78"); + let token_stream = TokenStream::new(vec![TokenTree::Ident(Token::new( + new_token_string.clone(), + TextSpan { start: 0, end: new_token_string.len() }, + ))]); let aux_data = AuxData::new(Vec::new()); ProcMacroResult::new(token_stream).with_aux_data(aux_data) } @@ -573,15 +573,15 @@ fn can_define_multiple_macros() { CairoPluginProjectBuilder::default() .name("other") .lib_rs(indoc! {r##" - use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, AuxData, PostProcessContext, post_process}; + use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, AuxData, PostProcessContext, post_process, TokenTree, Token, TextSpan}; #[attribute_macro] pub fn beautiful(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - let token_stream = TokenStream::new( - token_stream - .to_string() - .replace("90", "09") - ); + let new_token_string = token_stream.to_string().replace("90", "09"); + let token_stream = TokenStream::new(vec![TokenTree::Ident(Token::new( + new_token_string.clone(), + TextSpan { start: 0, end: new_token_string.len() }, + ))]); let aux_data = AuxData::new(Vec::new()); ProcMacroResult::new(token_stream).with_aux_data(aux_data) } @@ -777,7 +777,7 @@ fn can_resolve_full_path_markers() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, post_process, PostProcessContext}; + use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, post_process, PostProcessContext, TokenTree, Token, TextSpan}; #[attribute_macro] pub fn some(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { @@ -788,8 +788,14 @@ fn can_resolve_full_path_markers() { token_stream.to_string().replace("12", "34") ); - ProcMacroResult::new(TokenStream::new(code)) - .with_full_path_markers(full_path_markers) + ProcMacroResult::new(TokenStream::new(vec![TokenTree::Ident(Token::new( + code.clone(), + TextSpan { + start: 0, + end: code.len(), + }, + ))]) + ).with_full_path_markers(full_path_markers) } #[post_process] @@ -832,11 +838,17 @@ fn can_implement_inline_macro() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{ProcMacroResult, TokenStream, inline_macro}; + use cairo_lang_macro::{ProcMacroResult, TokenStream, inline_macro, TokenTree, Token, TextSpan}; #[inline_macro] pub fn some(_token_stream: TokenStream) -> ProcMacroResult { - ProcMacroResult::new(TokenStream::new("34".to_string())) + ProcMacroResult::new(TokenStream::new(vec![TokenTree::Ident(Token::new( + "34".to_string(), + TextSpan { + start: 0, + end: 2, + }, + ))])) } "##}) .build(&t); @@ -918,7 +930,7 @@ fn can_implement_derive_macro() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{derive_macro, ProcMacroResult, TokenStream}; + use cairo_lang_macro::{derive_macro, ProcMacroResult, TokenStream, TokenTree, Token, TextSpan}; #[derive_macro] pub fn custom_derive(token_stream: TokenStream) -> ProcMacroResult { @@ -935,13 +947,21 @@ fn can_implement_derive_macro() { .trim() .to_string(); - let token_stream = TokenStream::new(indoc::formatdoc!{r#" + let code = indoc::formatdoc!{r#" impl SomeImpl of Hello<{name}> {{ fn world(self: @{name}) -> u32 {{ 32 }} }} - "#}); + "#}; + + let token_stream = TokenStream::new(vec![TokenTree::Ident(Token::new( + code.clone(), + TextSpan { + start: 0, + end: code.len(), + }, + ))]); ProcMacroResult::new(token_stream) } @@ -991,37 +1011,58 @@ fn can_use_both_derive_and_attr() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{derive_macro, attribute_macro, ProcMacroResult, TokenStream}; + use cairo_lang_macro::{derive_macro, attribute_macro, ProcMacroResult, TokenStream, TokenTree, TextSpan, Token}; #[attribute_macro] pub fn first_attribute(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - ProcMacroResult::new(TokenStream::new( - token_stream.to_string() - .replace("SomeType", "OtherType") - )) + let new_token_string = token_stream.to_string().replace("SomeType", "OtherType"); + ProcMacroResult::new(TokenStream::new(vec![TokenTree::Ident(Token::new( + new_token_string.clone(), + TextSpan { + start: 0, + end: new_token_string.len(), + }, + ))])) } #[attribute_macro] pub fn second_attribute(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - let token_stream = TokenStream::new( - token_stream.to_string().replace("OtherType", "RenamedStruct") - ); - ProcMacroResult::new(TokenStream::new( - format!("#[derive(Drop)]\n{token_stream}") - )) + let code = token_stream.to_string().replace("OtherType", "RenamedStruct"); + let token_stream = TokenStream::new(vec![TokenTree::Ident(Token::new( + code.clone(), + TextSpan { + start: 0, + end: code.len(), + }, + ))]); + + let result_string = format!("#[derive(Drop)]\n{token_stream}"); + ProcMacroResult::new(TokenStream::new(vec![TokenTree::Ident(Token::new( + result_string.clone(), + TextSpan { + start: 0, + end: result_string.len(), + }, + ))])) } #[derive_macro] pub fn custom_derive(_token_stream: TokenStream) -> ProcMacroResult { - ProcMacroResult::new(TokenStream::new( - indoc::formatdoc!{r#" + let code = indoc::formatdoc!{r#" impl SomeImpl of Hello {{ fn world(self: @RenamedStruct) -> u32 {{ 32 }} }} - "#} - )) + "#}; + + ProcMacroResult::new(TokenStream::new(vec![TokenTree::Ident(Token::new( + code.clone(), + TextSpan { + start: 0, + end: code.len(), + }, + ))])) } "##}) .add_dep(r#"indoc = "*""#) @@ -1219,15 +1260,15 @@ fn can_be_expanded() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, derive_macro}; + use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro, derive_macro, TokenTree, Token, TextSpan}; #[attribute_macro] pub fn some(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - let token_stream = TokenStream::new( - token_stream - .to_string() - .replace("12", "34") - ); + let new_token_string = token_stream.to_string().replace("12", "34"); + let token_stream = TokenStream::new(vec![TokenTree::Ident(Token::new( + new_token_string.clone(), + TextSpan { start: 0, end: new_token_string.len() }, + ))]); ProcMacroResult::new(token_stream) } @@ -1246,13 +1287,18 @@ fn can_be_expanded() { .trim() .to_string(); - let token_stream = TokenStream::new(indoc::formatdoc!{r#" + let code = indoc::formatdoc!{r#" impl SomeImpl of Hello<{name}> {{ fn world(self: @{name}) -> u32 {{ 32 }} }} - "#}); + "#}; + + let token_stream = TokenStream::new(vec![TokenTree::Ident(Token::new( + code.clone(), + TextSpan { start: 0, end: code.len() }, + ))]); ProcMacroResult::new(token_stream) } @@ -1328,15 +1374,17 @@ fn can_expand_trait_inner_func_attrr() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{attribute_macro, ProcMacroResult, TokenStream}; + use cairo_lang_macro::{attribute_macro, ProcMacroResult, TokenStream, TokenTree, Token, TextSpan}; #[attribute_macro] pub fn some(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - ProcMacroResult::new(TokenStream::new( - token_stream.to_string() + let new_token_string = token_stream.to_string() .replace("hello", "world") - .replace("12", "34") - )) + .replace("12", "34"); + ProcMacroResult::new(TokenStream::new(vec![TokenTree::Ident(Token::new( + new_token_string.clone(), + TextSpan { start: 0, end: new_token_string.len() }, + ))])) } "##}) .build(&t); @@ -1388,14 +1436,15 @@ fn can_expand_impl_inner_func_attrr() { let t = temp.child("some"); CairoPluginProjectBuilder::default() .lib_rs(indoc! {r##" - use cairo_lang_macro::{attribute_macro, ProcMacroResult, TokenStream}; + use cairo_lang_macro::{attribute_macro, ProcMacroResult, TokenStream, Token, TokenTree, TextSpan}; #[attribute_macro] pub fn some(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult { - ProcMacroResult::new(TokenStream::new( - token_stream.to_string() - .replace("1", "2") - )) + let new_token_string = token_stream.to_string().replace("1", "2"); + ProcMacroResult::new(TokenStream::new(vec![TokenTree::Ident(Token::new( + new_token_string.clone(), + TextSpan { start: 0, end: new_token_string.len() }, + ))])) } "##}) .build(&t); diff --git a/scarb/tests/proc_macro_server.rs b/scarb/tests/proc_macro_server.rs index b01f865dc..c2a29d2fc 100644 --- a/scarb/tests/proc_macro_server.rs +++ b/scarb/tests/proc_macro_server.rs @@ -1,6 +1,6 @@ use assert_fs::prelude::PathChild; use assert_fs::TempDir; -use cairo_lang_macro::TokenStream; +use cairo_lang_macro::{TextSpan, Token, TokenStream, TokenTree}; use scarb_proc_macro_server_types::methods::defined_macros::DefinedMacros; use scarb_proc_macro_server_types::methods::defined_macros::DefinedMacrosParams; use scarb_proc_macro_server_types::methods::expand::ExpandAttribute; @@ -58,7 +58,14 @@ fn expand_attribute() { let output = input.replace(name, "very_new_name"); - ProcMacroResult::new(TokenStream::new(output)) + let span = TextSpan { start: 0, end: output.len() }; + ProcMacroResult::new( + TokenStream::new(vec![ + TokenTree::Ident( + Token::new(output, span) + ) + ]) + ) }} "##; @@ -82,14 +89,17 @@ fn expand_attribute() { .request_and_wait::(ExpandAttributeParams { attr: "rename_to_very_new_name".to_string(), args: TokenStream::empty(), - item: TokenStream::new("fn some_test_fn(){}".to_string()), + item: TokenStream::new(vec![TokenTree::Ident(Token::new( + "fn some_test_fn(){}".to_string(), + TextSpan::default(), + ))]), }) .unwrap(); assert_eq!(response.diagnostics, vec![]); assert_eq!( - response.token_stream, - TokenStream::new("fn very_new_name(){}".to_string()) + response.token_stream.to_string(), + "fn very_new_name(){}".to_string() ); } @@ -113,7 +123,10 @@ fn expand_derive() { let mut proc_macro_server = ProcMacroClient::new(&project); - let item = TokenStream::new("fn some_test_fn(){}".to_string()); + let item = TokenStream::new(vec![TokenTree::Ident(Token::new( + "fn some_test_fn(){}".to_string(), + TextSpan::default(), + ))]); let response = proc_macro_server .request_and_wait::(ExpandDeriveParams { @@ -124,8 +137,8 @@ fn expand_derive() { assert_eq!(response.diagnostics, vec![]); assert_eq!( - response.token_stream, - TokenStream::new("impl SomeImpl of SomeTrait {}".to_string()) + response.token_stream.to_string(), + "impl SomeImpl of SomeTrait {}".to_string() ); } @@ -137,7 +150,15 @@ fn expand_inline() { let replace_all_15_with_25 = r#" #[inline_macro] pub fn replace_all_15_with_25(token_stream: TokenStream) -> ProcMacroResult { - ProcMacroResult::new(TokenStream::new(token_stream.to_string().replace("15", "25"))) + let content = token_stream.to_string().replace("15", "25"); + let span = TextSpan { start: 0, end: content.len() }; + ProcMacroResult::new( + TokenStream::new(vec![ + TokenTree::Ident( + Token::new(content, span) + ) + ]) + ) } "#; @@ -159,15 +180,16 @@ fn expand_inline() { let response = proc_macro_server .request_and_wait::(ExpandInlineMacroParams { name: "replace_all_15_with_25".to_string(), - args: TokenStream::new( + args: TokenStream::new(vec![TokenTree::Ident(Token::new( "struct A { field: 15 , other_field: macro_call!(12)}".to_string(), - ), + TextSpan::default(), + ))]), }) .unwrap(); assert_eq!(response.diagnostics, vec![]); assert_eq!( - response.token_stream, - TokenStream::new("struct A { field: 25 , other_field: macro_call!(12)}".to_string()) + response.token_stream.to_string(), + "struct A { field: 25 , other_field: macro_call!(12)}".to_string() ); } diff --git a/utils/scarb-test-support/src/proc_macro_server.rs b/utils/scarb-test-support/src/proc_macro_server.rs index 12e93176f..4586b179c 100644 --- a/utils/scarb-test-support/src/proc_macro_server.rs +++ b/utils/scarb-test-support/src/proc_macro_server.rs @@ -19,7 +19,7 @@ use std::process::Stdio; pub const SIMPLE_MACROS: &str = r#" use cairo_lang_macro::{ ProcMacroResult, - TokenStream, + TokenStream, TokenTree, Token, TextSpan, attribute_macro, inline_macro, derive_macro, @@ -40,7 +40,15 @@ pub fn inline_some(token_stream: TokenStream) -> ProcMacroResult { #[derive_macro] fn some_derive(_token_stream: TokenStream)-> ProcMacroResult { - ProcMacroResult::new(TokenStream::new("impl SomeImpl of SomeTrait {}".to_string())) + let content = "impl SomeImpl of SomeTrait {}".to_string(); + let span = TextSpan { start: 0, end: content.len() }; + ProcMacroResult::new( + TokenStream::new(vec![ + TokenTree::Ident( + Token::new(content, span) + ) + ]) + ) } "#;