Rework internal TokenStream representation

plugins/cairo-lang-macro-stable/src/lib.rs

@@ -1,11 +1,30 @@
 use crate::ffi::{StableOption, StableSlice};
-use std::ffi::CStr;
 use std::num::NonZeroU8;
 use std::os::raw::c_char;
 use std::ptr::NonNull;
 
 pub mod ffi;
 
+#[repr(C)]
+#[derive(Debug)]
+pub struct StableToken {
+    pub span: StableTextSpan,
+    pub content: *mut c_char,
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct StableTextSpan {
+    pub start: usize,
+    pub end: usize,
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub enum StableTokenTree {
+    Ident(StableToken),
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct StableExpansion {
@@ -23,7 +42,7 @@ pub type StableExpansionsList = StableSlice<StableExpansion>;
 #[repr(C)]
 #[derive(Debug)]
 pub struct StableTokenStream {
-    pub value: *mut c_char,
+    pub tokens: StableSlice<StableTokenTree>,
     pub metadata: StableTokenStreamMetadata,
 }
 
@@ -76,17 +95,6 @@ pub struct StableResultWrapper {
     pub output: StableProcMacroResult,
 }
 
-impl StableTokenStream {
-    /// Convert to String.
-    ///
-    /// # Safety
-    pub unsafe fn to_string(&self) -> String {
-        // Note that this does not deallocate the c-string.
-        // The memory must still be freed with `CString::from_raw`.
-        CStr::from_ptr(self.value).to_string_lossy().to_string()
-    }
-}
-
 #[repr(C)]
 pub struct StablePostProcessContext {
     pub aux_data: StableSlice<StableAuxData>,

plugins/cairo-lang-macro/Cargo.toml

@@ -14,7 +14,7 @@ repository.workspace = true
 
 [dependencies]
 cairo-lang-macro-attributes = "0.1"
-cairo-lang-macro-stable = "1"
+cairo-lang-macro-stable = { path = "../cairo-lang-macro-stable" }
 linkme.workspace = true
 serde = { workspace = true, optional = true }
 

plugins/cairo-lang-macro/src/types/conversion.rs

@@ -1,12 +1,12 @@
 use crate::{
     AuxData, Diagnostic, ExpansionDefinition, FullPathMarker, PostProcessContext, ProcMacroResult,
-    Severity, TokenStream, TokenStreamMetadata,
+    Severity, TextSpan, Token, TokenStream, TokenStreamMetadata, TokenTree,
 };
 use cairo_lang_macro_stable::ffi::StableSlice;
 use cairo_lang_macro_stable::{
     StableAuxData, StableDiagnostic, StableExpansion, StableFullPathMarker,
-    StablePostProcessContext, StableProcMacroResult, StableSeverity, StableTokenStream,
-    StableTokenStreamMetadata,
+    StablePostProcessContext, StableProcMacroResult, StableSeverity, StableTextSpan, StableToken,
+    StableTokenStream, StableTokenStreamMetadata, StableTokenTree,
 };
 use std::ffi::{c_char, CStr, CString};
 use std::num::NonZeroU8;
@@ -90,15 +90,120 @@ impl ProcMacroResult {
     }
 }
 
+impl TextSpan {
+    /// Convert to FFI-safe representation.
+    #[doc(hidden)]
+    pub fn into_stable(self) -> StableTextSpan {
+        StableTextSpan {
+            start: self.start,
+            end: self.end,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn from_stable(span: &StableTextSpan) -> Self {
+        Self {
+            start: span.start,
+            end: span.end,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn from_owned_stable(span: StableTextSpan) -> Self {
+        Self {
+            start: span.start,
+            end: span.end,
+        }
+    }
+}
+
+impl Token {
+    /// Convert to FFI-safe representation.
+    #[doc(hidden)]
+    pub fn into_stable(self) -> StableToken {
+        let cstr = CString::new(self.content.as_bytes()).unwrap();
+        StableToken {
+            span: self.span.into_stable(),
+            content: cstr.into_raw(),
+        }
+    }
+
+    /// Convert to native Rust representation, without taking the ownership of the string.
+    ///
+    /// Note that you still need to free the memory by calling `from_owned_stable`.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub unsafe fn from_stable(token: &StableToken) -> Self {
+        Self {
+            content: from_raw_cstr(token.content),
+            span: TextSpan::from_stable(&token.span),
+        }
+    }
+
+    /// Convert to native Rust representation, with taking the ownership of the string.
+    ///
+    /// Useful when you need to free the allocated memory.
+    /// Only use on the same side of FFI-barrier, where the memory has been allocated.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub unsafe fn from_owned_stable(token: StableToken) -> Self {
+        Self {
+            content: from_raw_cstring(token.content),
+            span: TextSpan::from_owned_stable(token.span),
+        }
+    }
+}
+
+impl TokenTree {
+    /// Convert to FFI-safe representation.
+    #[doc(hidden)]
+    pub fn into_stable(self) -> StableTokenTree {
+        match self {
+            Self::Ident(token) => StableTokenTree::Ident(token.into_stable()),
+        }
+    }
+
+    /// Convert to native Rust representation, without taking the ownership of the string.
+    ///
+    /// Note that you still need to free the memory by calling `from_owned_stable`.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub unsafe fn from_stable(token_tree: &StableTokenTree) -> Self {
+        match token_tree {
+            StableTokenTree::Ident(token) => Self::Ident(Token::from_stable(token)),
+        }
+    }
+
+    /// Convert to native Rust representation, with taking the ownership of the string.
+    ///
+    /// Useful when you need to free the allocated memory.
+    /// Only use on the same side of FFI-barrier, where the memory has been allocated.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub unsafe fn from_owned_stable(token_tree: StableTokenTree) -> Self {
+        match token_tree {
+            StableTokenTree::Ident(token) => Self::Ident(Token::from_owned_stable(token)),
+        }
+    }
+}
+
 impl TokenStream {
     /// Convert to FFI-safe representation.
     ///
     /// # Safety
     #[doc(hidden)]
     pub fn into_stable(self) -> StableTokenStream {
-        let cstr = CString::new(self.value).unwrap();
+        let tokens = self
+            .tokens
+            .into_iter()
+            .map(|token| token.into_stable())
+            .collect::<Vec<_>>();
         StableTokenStream {
-            value: cstr.into_raw(),
+            tokens: StableSlice::new(tokens),
             metadata: self.metadata.into_stable(),
         }
     }
@@ -110,8 +215,13 @@ impl TokenStream {
     /// # Safety
     #[doc(hidden)]
     pub unsafe fn from_stable(token_stream: &StableTokenStream) -> Self {
+        let (ptr, n) = token_stream.tokens.raw_parts();
+        let tokens = slice::from_raw_parts(ptr, n)
+            .iter()
+            .map(|token_tree| TokenTree::from_stable(token_tree))
+            .collect::<Vec<_>>();
         Self {
-            value: from_raw_cstr(token_stream.value),
+            tokens,
             metadata: TokenStreamMetadata::from_stable(&token_stream.metadata),
         }
     }
@@ -124,8 +234,13 @@ impl TokenStream {
     /// # Safety
     #[doc(hidden)]
     pub unsafe fn from_owned_stable(token_stream: StableTokenStream) -> Self {
+        let tokens = token_stream.tokens.into_owned();
+        let tokens = tokens
+            .into_iter()
+            .map(|token_tree| TokenTree::from_owned_stable(token_tree))
+            .collect::<Vec<_>>();
         Self {
-            value: from_raw_cstring(token_stream.value),
+            tokens,
             metadata: TokenStreamMetadata::from_owned_stable(token_stream.metadata),
         }
     }

plugins/cairo-lang-macro/src/types/mod.rs

@@ -1,10 +1,11 @@
-use std::fmt::Display;
 use std::vec::IntoIter;
 
 mod conversion;
 mod expansions;
+mod token;
 
 pub use expansions::*;
+pub use token::*;
 
 /// Result of procedural macro code generation.
 #[derive(Debug, Clone)]
@@ -15,78 +16,6 @@ pub struct ProcMacroResult {
     pub full_path_markers: Vec<String>,
 }
 
-/// An abstract stream of Cairo tokens.
-///
-/// This is both input and part of an output of a procedural macro.
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)]
-pub struct TokenStream {
-    value: String,
-    metadata: TokenStreamMetadata,
-}
-
-/// Metadata of [`TokenStream`].
-///
-/// This struct can be used to describe the origin of the [`TokenStream`].
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)]
-pub struct TokenStreamMetadata {
-    /// The path to the file from which the [`TokenStream`] has been created.
-    pub original_file_path: Option<String>,
-    /// ID of the file from which the [`TokenStream`] has been created.
-    ///
-    /// It is guaranteed, that the `file_id` will be unique for each file.
-    pub file_id: Option<String>,
-}
-
-impl TokenStream {
-    #[doc(hidden)]
-    pub fn new(value: String) -> Self {
-        Self {
-            value,
-            metadata: TokenStreamMetadata::default(),
-        }
-    }
-
-    #[doc(hidden)]
-    pub fn empty() -> Self {
-        Self::new("".to_string())
-    }
-
-    #[doc(hidden)]
-    pub fn with_metadata(mut self, metadata: TokenStreamMetadata) -> Self {
-        self.metadata = metadata;
-        self
-    }
-
-    /// Get `[TokenStreamMetadata`] associated with this [`TokenStream`].
-    ///
-    /// The metadata struct can be used to describe the [`TokenStream`] origin.
-    pub fn metadata(&self) -> &TokenStreamMetadata {
-        &self.metadata
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.to_string().is_empty()
-    }
-}
-
-impl Display for TokenStream {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.value)
-    }
-}
-
-impl TokenStreamMetadata {
-    #[doc(hidden)]
-    pub fn new(file_path: impl ToString, file_id: impl ToString) -> Self {
-        Self {
-            original_file_path: Some(file_path.to_string()),
-            file_id: Some(file_id.to_string()),
-        }
-    }
-}
-
 /// **Auxiliary data** returned by procedural macro code generation.
 ///
 /// This struct can be used to collect additional information from the Cairo source code of
@@ -101,7 +30,7 @@ impl TokenStreamMetadata {
 /// For instance, auxiliary data can be serialized as JSON.
 ///
 /// ```
-/// use cairo_lang_macro::{AuxData, ProcMacroResult, TokenStream, attribute_macro, post_process, PostProcessContext};
+/// use cairo_lang_macro::{AuxData, ProcMacroResult, TokenStream, TokenTree, Token, TextSpan, attribute_macro, post_process, PostProcessContext};
 /// use serde::{Serialize, Deserialize};
 /// #[derive(Debug, Serialize, Deserialize)]
 /// struct SomeAuxDataFormat {
@@ -110,11 +39,16 @@ impl TokenStreamMetadata {
 ///
 /// #[attribute_macro]
 /// pub fn some_macro(_attr: TokenStream, token_stream: TokenStream) -> ProcMacroResult {
-///     let token_stream = TokenStream::new(
-///         token_stream.to_string()
-///         // Remove macro call to avoid infinite loop.
-///         .replace("#[some]", "")
-///     );
+///     // Remove macro call to avoid infinite loop.
+///     let code = token_stream.to_string().replace("#[some]", "");
+///     let token_stream = TokenStream::new(vec![
+///         TokenTree::Ident(
+///             Token::new(
+///                 code.clone(),
+///                 TextSpan::new(0, code.len())
+///             )
+///         )
+///     ]);
 ///     let value = SomeAuxDataFormat { some_message: "Hello from some macro!".to_string() };
 ///     let value = serde_json::to_string(&value).unwrap();
 ///     let value: Vec<u8> = value.into_bytes();