Store file path and file id in token stream metadata

commit-id:98fd5a13

plugins/cairo-lang-macro-stable/src/lib.rs

@@ -1,24 +1,45 @@
 use crate::ffi::StableSlice;
-use std::ffi::{CStr, CString};
+use std::ffi::CStr;
 use std::num::NonZeroU8;
 use std::os::raw::c_char;
+use std::ptr::NonNull;
 
 pub mod ffi;
 
+/// An option.
+///
+/// This struct implements FFI-safe stable ABI.
+#[repr(C)]
+#[derive(Debug)]
+pub enum StableOption<T> {
+    None,
+    Some(T),
+}
+
 /// Token stream.
 ///
 /// This struct implements FFI-safe stable ABI.
 #[repr(C)]
 #[derive(Debug)]
-pub struct StableTokenStream(*mut c_char);
+pub struct StableTokenStream {
+    pub value: *mut c_char,
+    pub metadata: StableTokenStreamMetadata,
+}
 
+/// Token stream metadata.
+///
+/// This struct implements FFI-safe stable ABI.
 #[repr(C)]
 #[derive(Debug)]
-pub enum StableAuxData {
-    None,
-    Some(StableSlice<u8>),
+pub struct StableTokenStreamMetadata {
+    pub original_file_path: Option<NonNull<c_char>>,
 }
 
+/// Auxiliary data returned by the procedural macro.
+///
+/// This struct implements FFI-safe stable ABI.
+pub type StableAuxData = StableOption<StableSlice<u8>>;
+
 /// Diagnostic returned by the procedural macro.
 ///
 /// This struct implements FFI-safe stable ABI.
@@ -63,29 +84,12 @@ pub struct StableResultWrapper {
 }
 
 impl StableTokenStream {
-    pub fn new(s: *mut c_char) -> Self {
-        Self(s)
-    }
-
     /// Convert to String.
     ///
     /// # Safety
     pub unsafe fn to_string(&self) -> String {
         // Note that this does not deallocate the c-string.
         // The memory must still be freed with `CString::from_raw`.
-        CStr::from_ptr(self.0).to_string_lossy().to_string()
-    }
-
-    pub fn into_owned_string(self) -> String {
-        unsafe { raw_to_string(self.0) }
-    }
-}
-
-unsafe fn raw_to_string(raw: *mut c_char) -> String {
-    if raw.is_null() {
-        String::default()
-    } else {
-        let cstr = CString::from_raw(raw);
-        cstr.to_string_lossy().to_string()
+        CStr::from_ptr(self.value).to_string_lossy().to_string()
     }
 }

plugins/cairo-lang-macro/Cargo.toml

@@ -16,3 +16,4 @@ repository.workspace = true
 cairo-lang-macro-attributes = { path = "../cairo-lang-macro-attributes" }
 cairo-lang-macro-stable = { path = "../cairo-lang-macro-stable" }
 linkme.workspace = true
+scarb-stable-hash = { path = "../../utils/scarb-stable-hash" }

plugins/cairo-lang-macro/src/lib.rs

@@ -4,10 +4,13 @@ pub use linkme::distributed_slice;
 use cairo_lang_macro_stable::ffi::StableSlice;
 use cairo_lang_macro_stable::{
     StableAuxData, StableDiagnostic, StableProcMacroResult, StableSeverity, StableTokenStream,
+    StableTokenStreamMetadata,
 };
+use scarb_stable_hash::short_hash;
 use std::ffi::{c_char, CStr, CString};
 use std::fmt::Display;
 use std::num::NonZeroU8;
+use std::ptr::NonNull;
 use std::slice;
 use std::vec::IntoIter;
 
@@ -65,18 +68,59 @@ pub enum ProcMacroResult {
 }
 
 #[derive(Debug, Default, Clone)]
-pub struct TokenStream(String);
+pub struct TokenStream {
+    value: String,
+    metadata: TokenStreamMetadata,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct TokenStreamMetadata {
+    original_file_path: Option<String>,
+    file_id: Option<String>,
+}
 
 impl TokenStream {
     #[doc(hidden)]
-    pub fn new(s: String) -> Self {
-        Self(s)
+    pub fn new(value: String) -> Self {
+        Self {
+            value,
+            metadata: TokenStreamMetadata::default(),
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn with_file_path(mut self, file_path: impl ToString) -> Self {
+        self.metadata = self.metadata.with_file_path(file_path);
+        self
+    }
+
+    pub fn metadata(&self) -> &TokenStreamMetadata {
+        &self.metadata
     }
 }
 
 impl Display for TokenStream {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.0)
+        write!(f, "{}", self.value)
+    }
+}
+
+impl TokenStreamMetadata {
+    #[doc(hidden)]
+    pub fn new(file_path: Option<String>) -> Self {
+        let s = Self::default();
+        if let Some(file_path) = file_path {
+            s.with_file_path(file_path)
+        } else {
+            s
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn with_file_path(mut self, file_path: impl ToString) -> Self {
+        self.original_file_path = Some(file_path.to_string());
+        self.file_id = Some(short_hash(file_path.to_string()));
+        self
     }
 }
 
@@ -331,8 +375,11 @@ impl TokenStream {
     /// # Safety
     #[doc(hidden)]
     pub fn into_stable(self) -> StableTokenStream {
-        let cstr = CString::new(self.0).unwrap();
-        StableTokenStream::new(cstr.into_raw())
+        let cstr = CString::new(self.value).unwrap();
+        StableTokenStream {
+            value: cstr.into_raw(),
+            metadata: self.metadata.into_stable(),
+        }
     }
 
     /// Convert to native Rust representation, without taking the ownership of the string.
@@ -342,7 +389,10 @@ impl TokenStream {
     /// # Safety
     #[doc(hidden)]
     pub unsafe fn from_stable(token_stream: &StableTokenStream) -> Self {
-        Self::new(token_stream.to_string())
+        Self {
+            value: from_raw_cstr(token_stream.value),
+            metadata: TokenStreamMetadata::from_stable(&token_stream.metadata),
+        }
     }
 
     /// Convert to native Rust representation, with taking the ownership of the string.
@@ -353,7 +403,62 @@ impl TokenStream {
     /// # Safety
     #[doc(hidden)]
     pub unsafe fn from_owned_stable(token_stream: StableTokenStream) -> Self {
-        Self::new(token_stream.into_owned_string())
+        Self {
+            value: from_raw_cstring(token_stream.value),
+            metadata: TokenStreamMetadata::from_owned_stable(token_stream.metadata),
+        }
+    }
+}
+
+impl TokenStreamMetadata {
+    /// Convert to FFI-safe representation.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub fn into_stable(self) -> StableTokenStreamMetadata {
+        if let Some(original_file_path) = self.original_file_path {
+            let cstr = CString::new(original_file_path).unwrap();
+            StableTokenStreamMetadata {
+                original_file_path: NonNull::new(cstr.into_raw()),
+            }
+        } else {
+            StableTokenStreamMetadata {
+                original_file_path: None,
+            }
+        }
+    }
+
+    /// Convert to native Rust representation, without taking the ownership of the string.
+    ///
+    /// Note that you still need to free the memory by calling `from_owned_stable`.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub unsafe fn from_stable(metadata: &StableTokenStreamMetadata) -> Self {
+        match &metadata.original_file_path {
+            Some(raw) => {
+                let original_file_path = from_raw_cstr(raw.as_ptr());
+                Self::new(Some(original_file_path))
+            }
+            None => Self::new(None),
+        }
+    }
+
+    /// Convert to native Rust representation, with taking the ownership of the string.
+    ///
+    /// Useful when you need to free the allocated memory.
+    /// Only use on the same side of FFI-barrier, where the memory has been allocated.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub unsafe fn from_owned_stable(metadata: StableTokenStreamMetadata) -> Self {
+        match metadata.original_file_path {
+            Some(raw) => {
+                let original_file_path = from_raw_cstring(raw.as_ptr());
+                Self::new(Some(original_file_path))
+            }
+            None => Self::new(None),
+        }
     }
 }
 
@@ -494,3 +599,37 @@ unsafe fn from_raw_cstr(raw: *mut c_char) -> String {
         cstr.to_string_lossy().to_string()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::TokenStream;
+
+    #[test]
+    fn new_token_stream_metadata_empty() {
+        let token_stream = TokenStream::new("".to_string());
+        assert!(token_stream.metadata.file_id.is_none());
+        assert!(token_stream.metadata.original_file_path.is_none());
+    }
+
+    #[test]
+    fn token_stream_metadata_defines_file_id() {
+        let token_stream = TokenStream::new("".to_string());
+        let some_path = "/some/file/path/file.rs".to_string();
+
+        let token_stream = token_stream.with_file_path(some_path.clone());
+        assert!(token_stream.metadata.file_id.is_some());
+        assert_eq!(
+            token_stream.metadata.original_file_path.clone().unwrap(),
+            some_path.clone()
+        );
+        let first_file_id = token_stream.metadata.file_id.clone().unwrap();
+
+        let token_stream = TokenStream::new("".to_string());
+        let token_stream = token_stream.with_file_path(some_path);
+        assert_eq!(
+            token_stream.metadata.file_id.clone().unwrap(),
+            first_file_id,
+            "file_id should be stable in regard to file path"
+        );
+    }
+}

scarb/src/compiler/plugin/proc_macro/host.rs

@@ -203,8 +203,9 @@ impl MacroPlugin for ProcMacroHostPlugin {
             .chain(self.handle_attribute(db, item_ast.clone()))
             .chain(self.handle_derive(db, item_ast.clone()));
         let stable_ptr = item_ast.clone().stable_ptr().untyped();
+        let file_path = stable_ptr.file_id(db).full_path(db.upcast());
 
-        let mut token_stream = TokenStream::from_item_ast(db, item_ast);
+        let mut token_stream = TokenStream::from_item_ast(db, item_ast).with_file_path(file_path);
         let mut aux_data: Option<ProcMacroAuxData> = None;
         let mut modified = false;
         let mut all_diagnostics: Vec<Diagnostic> = Vec::new();

scarb/tests/build_cairo_plugin.rs

@@ -519,7 +519,7 @@ fn can_return_aux_data_from_plugin() {
         .build(&project);
 
     Scarb::quick_snapbox()
-        .arg("cairo-run")
+        .arg("build")
         // Disable output from Cargo.
         .env("CARGO_TERM_QUIET", "true")
         .current_dir(&project)
@@ -530,7 +530,50 @@ fn can_return_aux_data_from_plugin() {
             [..]Compiling hello v1.0.0 ([..]Scarb.toml)
             [SomeMacroDataFormat { msg: "Hello from some macro!" }]
             [..]Finished release target(s) in [..]
-            [..]Running hello
-            [..]Run completed successfully, returning [..]
+        "#});
+}
+
+#[test]
+fn can_read_token_stream_metadata() {
+    let temp = TempDir::new().unwrap();
+    let t = temp.child("some");
+    simple_project_with_code(
+        &t,
+        indoc! {r##"
+        use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro};
+
+        #[attribute_macro]
+        pub fn some_macro(token_stream: TokenStream) -> ProcMacroResult {
+            println!("{:?}", token_stream.metadata());
+            ProcMacroResult::Leave { diagnostics: Vec::new() }
+        }
+
+        "##},
+    );
+
+    let project = temp.child("hello");
+    ProjectBuilder::start()
+        .name("hello")
+        .version("1.0.0")
+        .dep_starknet()
+        .dep("some", &t)
+        .lib_cairo(indoc! {r#"
+            #[some]
+            fn main() -> felt252 { 12 }
+        "#})
+        .build(&project);
+
+    Scarb::quick_snapbox()
+        .arg("build")
+        // Disable output from Cargo.
+        .env("CARGO_TERM_QUIET", "true")
+        .current_dir(&project)
+        .assert()
+        .success()
+        .stdout_matches(indoc! {r#"
+            [..]Compiling some v1.0.0 ([..]Scarb.toml)
+            [..]Compiling hello v1.0.0 ([..]Scarb.toml)
+            TokenStreamMetadata { original_file_path: Some("[..]lib.cairo"), file_id: Some("[..]") }
+            [..]Finished release target(s) in [..]
         "#});
 }