IDL: Use cargo-expand to expand macros in code (#23)

cargo-expand is already used for the `anchor expand` subcommand.
In contrast to using `cargo +nightly rustc --profile=check --
-Zunpretty=expanded`, it works without problems in all of our
Anchor programs.

The code for using cargo-expand used to be a part of anchor-cli
crate, but since we had to use it in anchor-syn, this change moves
it to a separate crate.

Cargo.toml

@@ -12,6 +12,7 @@ members = [
     "lang",
     "lang/attribute/*",
     "lang/derive/*",
+    "lang/expand",
     "lang/syn",
     "spl",
 ]

cli/Cargo.toml

@@ -18,6 +18,7 @@ default = []
 
 [dependencies]
 anchor-client = { path = "../client", version = "0.28.0", package = "light-anchor-client" }
+anchor-expand = { path = "../lang/expand", version = "0.28.0", package = "light-anchor-expand" }
 anchor-lang = { path = "../lang", version = "0.28.0", package = "light-anchor-lang" }
 anchor-syn = { path = "../lang/syn", features = ["event-cpi", "idl", "init-if-needed"], version = "0.28.0", package = "light-anchor-syn" }
 anyhow = "1.0.32"

cli/src/lib.rs

@@ -4,6 +4,7 @@ use crate::config::{
     STARTUP_WAIT,
 };
 use anchor_client::Cluster;
+use anchor_expand::expand_program;
 use anchor_lang::idl::{IdlAccount, IdlInstruction, ERASED_AUTHORITY};
 use anchor_lang::{AccountDeserialize, AnchorDeserialize, AnchorSerialize};
 use anchor_syn::idl::{EnumFields, Idl, IdlType, IdlTypeDefinitionTy};
@@ -856,12 +857,17 @@ pub fn expand(
             expand_all(&workspace_cfg, expansions_path, cargo_args)
         }
         // Reaching this arm means Cargo.toml belongs to a single package. Expand it.
-        Some((_, cargo)) => expand_program(
-            // If we found Cargo.toml, it must be in a directory so unwrap is safe
-            cargo.path().parent().unwrap().to_path_buf(),
-            expansions_path,
-            cargo_args,
-        ),
+        Some((_, cargo)) => {
+            expand_program(
+                // If we found Cargo.toml, it must be in a directory so unwrap is safe
+                cargo.path().parent().unwrap().to_path_buf(),
+                &cargo.package().name,
+                cargo.package().version(),
+                Some(expansions_path),
+                cargo_args,
+            )?;
+            Ok(())
+        }
     }
 }
 
@@ -872,60 +878,19 @@ fn expand_all(
 ) -> Result<()> {
     let cur_dir = std::env::current_dir()?;
     for p in workspace_cfg.get_rust_program_list()? {
-        expand_program(p, expansions_path.clone(), cargo_args)?;
+        let cargo = Manifest::from_path(p.join("Cargo.toml"))?;
+        expand_program(
+            p,
+            &cargo.package().name,
+            cargo.package().version(),
+            Some(expansions_path.clone()),
+            cargo_args,
+        )?;
     }
     std::env::set_current_dir(cur_dir)?;
     Ok(())
 }
 
-fn expand_program(
-    program_path: PathBuf,
-    expansions_path: PathBuf,
-    cargo_args: &[String],
-) -> Result<()> {
-    let cargo = Manifest::from_path(program_path.join("Cargo.toml"))
-        .map_err(|_| anyhow!("Could not find Cargo.toml for program"))?;
-
-    let target_dir_arg = {
-        let mut target_dir_arg = OsString::from("--target-dir=");
-        target_dir_arg.push(expansions_path.join("expand-target"));
-        target_dir_arg
-    };
-
-    let package_name = &cargo
-        .package
-        .as_ref()
-        .ok_or_else(|| anyhow!("Cargo config is missing a package"))?
-        .name;
-    let program_expansions_path = expansions_path.join(package_name);
-    fs::create_dir_all(&program_expansions_path)?;
-
-    let exit = std::process::Command::new("cargo")
-        .arg("expand")
-        .arg(target_dir_arg)
-        .arg(&format!("--package={package_name}"))
-        .args(cargo_args)
-        .stderr(Stdio::inherit())
-        .output()
-        .map_err(|e| anyhow::format_err!("{}", e.to_string()))?;
-    if !exit.status.success() {
-        eprintln!("'anchor expand' failed. Perhaps you have not installed 'cargo-expand'? https://github.com/dtolnay/cargo-expand#installation");
-        std::process::exit(exit.status.code().unwrap_or(1));
-    }
-
-    let version = cargo.version();
-    let time = chrono::Utc::now().to_string().replace(' ', "_");
-    let file_path = program_expansions_path.join(format!("{package_name}-{version}-{time}.rs"));
-    fs::write(&file_path, &exit.stdout).map_err(|e| anyhow::format_err!("{}", e.to_string()))?;
-
-    println!(
-        "Expanded {} into file {}\n",
-        package_name,
-        file_path.to_string_lossy()
-    );
-    Ok(())
-}
-
 #[allow(clippy::too_many_arguments)]
 pub fn build(
     cfg_override: &ConfigOverride,
@@ -1840,9 +1805,10 @@ fn extract_idl(
     let (crate_root, cargo) = Manifest::discover_from_path(manifest_from_path)?
         .ok_or_else(|| anyhow!("Cargo.toml not found"))?;
     anchor_syn::idl::file::parse(
+        &cargo.package().name,
         crate_root,
         cargo.path().to_path_buf(),
-        cargo.version(),
+        &cargo.version(),
         cfg.features.seeds,
         no_docs,
         !(cfg.features.skip_lint || skip_lint),

lang/expand/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "light-anchor-expand"
+version = "0.28.0"
+authors = ["Anchor Maintainers <[email protected]>"]
+repository = "https://github.com/coral-xyz/anchor"
+license = "Apache-2.0"
+description = "Utility library for calling cargo-expand"
+rust-version = "1.60"
+edition = "2021"
+
+[dependencies]
+anyhow = "1"
+cargo_toml = "0.13.0"
+chrono = "0.4.19"
+heck = "0.4.0"
+serde_json = "1"

lang/expand/src/lib.rs

@@ -0,0 +1,56 @@
+use std::{ffi::OsString, fs, path::PathBuf, process::Stdio};
+
+use anyhow::Result;
+
+pub fn expand_program(
+    root: PathBuf,
+    package_name: &str,
+    version: &str,
+    expansions_path: Option<PathBuf>,
+    cargo_args: &[String],
+) -> Result<Vec<u8>> {
+    let target_dir_arg = match expansions_path {
+        Some(ref expansions_path) => {
+            let mut target_dir_arg = OsString::from("--target-dir=");
+            target_dir_arg.push(expansions_path.join("expand-target"));
+            Some(target_dir_arg)
+        }
+        None => None,
+    };
+
+    let mut cmd = std::process::Command::new("cargo");
+    let cmd = cmd.arg("expand");
+    if let Some(target_dir_arg) = target_dir_arg {
+        cmd.arg(target_dir_arg);
+    }
+    let exit = cmd
+        .current_dir(root)
+        .arg(&format!("--package={package_name}"))
+        .args(cargo_args)
+        .stderr(Stdio::inherit())
+        .output()
+        .map_err(|e| anyhow::format_err!("{}", e.to_string()))?;
+    if !exit.status.success() {
+        eprintln!("'cargo expand' failed. Perhaps you have not installed 'cargo-expand'? https://github.com/dtolnay/cargo-expand#installation");
+        std::process::exit(exit.status.code().unwrap_or(1));
+    }
+
+    if let Some(ref expansions_path) = expansions_path {
+        let program_expansions_path = expansions_path.join(package_name);
+        fs::create_dir_all(&program_expansions_path)?;
+
+        // let version = cargo.version();
+        let time = chrono::Utc::now().to_string().replace(' ', "_");
+        let file_path = program_expansions_path.join(format!("{package_name}-{version}-{time}.rs"));
+        fs::write(&file_path, &exit.stdout)
+            .map_err(|e| anyhow::format_err!("{}", e.to_string()))?;
+
+        println!(
+            "Expanded {} into file {}\n",
+            package_name,
+            file_path.to_string_lossy()
+        );
+    }
+
+    Ok(exit.stdout)
+}

lang/syn/Cargo.toml

@@ -19,6 +19,7 @@ seeds = []
 event-cpi = []
 
 [dependencies]
+anchor-expand = { path = "../expand", package = "light-anchor-expand" }
 anyhow = "1"
 bs58 = "0.5"
 heck = "0.3"

lang/syn/src/idl/file.rs

@@ -25,14 +25,15 @@ const ERROR_CODE_OFFSET: u32 = 6000;
 
 // Parse an entire interface file.
 pub fn parse(
+    package_name: &str,
     crate_root: PathBuf,
     cargo_path: PathBuf,
-    version: String,
+    version: &str,
     seeds_feature: bool,
     no_docs: bool,
     safety_checks: bool,
 ) -> Result<Option<Idl>> {
-    let ctx = CrateContext::parse(crate_root, &None)?;
+    let ctx = CrateContext::parse(crate_root, package_name, version, &None)?;
     if safety_checks {
         ctx.safety_checks()?;
     }
@@ -210,7 +211,12 @@ pub fn parse(
             };
 
             let mut parsed_types = HashSet::new();
-            let crate_ctx = CrateContext::parse(&dependency.path, &dependency.features)?;
+            let crate_ctx = CrateContext::parse(
+                &dependency.path,
+                &dependency.package_name,
+                &dependency.version,
+                &dependency.features,
+            )?;
             let (_, account_serialize_impls, borsh_serialize_impls) = parse_impls(&crate_ctx);
             let ty_defs = parse_ty_defs(
                 &crate_ctx,
@@ -238,7 +244,7 @@ pub fn parse(
         .collect::<Result<Vec<IdlConst>, _>>()?;
 
     Ok(Some(Idl {
-        version,
+        version: version.to_string(),
         name: p.name.to_string(),
         docs: p.docs.clone(),
         instructions,
@@ -257,7 +263,9 @@ pub fn parse(
 
 /// Dependency of a Rust crate (an another crate).
 struct CargoDependency {
+    package_name: String,
     path: PathBuf,
+    version: String,
     features: Option<Vec<String>>,
 }
 
@@ -302,7 +310,9 @@ where
                 (
                     package.name.clone(),
                     CargoDependency {
+                        package_name: package.name.clone(),
                         path: cache_dir.to_path_buf().into_std_path_buf(),
+                        version: package.version.to_string(),
                         features: dependency_features.get(&package.name).cloned(),
                     },
                 )

lang/syn/src/parser/context.rs

@@ -1,9 +1,6 @@
-use std::{
-    collections::BTreeMap,
-    path::Path,
-    process::{Command, Stdio},
-};
+use std::{collections::BTreeMap, path::Path};
 
+use anchor_expand::expand_program;
 use anyhow::anyhow;
 use syn::{Ident, ImplItem, ImplItemConst, Type, TypePath};
 
@@ -51,10 +48,12 @@ impl CrateContext {
 
     pub fn parse(
         root: impl AsRef<Path>,
+        package_name: &str,
+        version: &str,
         features: &Option<Vec<String>>,
     ) -> Result<Self, anyhow::Error> {
         Ok(CrateContext {
-            modules: ParsedModule::parse_recursive(root.as_ref(), features)?,
+            modules: ParsedModule::parse_recursive(root.as_ref(), package_name, version, features)?,
         })
     }
 
@@ -115,29 +114,19 @@ struct ParsedModule {
 impl ParsedModule {
     fn parse_recursive(
         root: &Path,
+        package_name: &str,
+        version: &str,
         features: &Option<Vec<String>>,
     ) -> Result<BTreeMap<String, ParsedModule>, anyhow::Error> {
         let mut modules = BTreeMap::new();
 
-        let mut args = vec![
-            "+nightly".to_owned(),
-            "rustc".to_owned(),
-            "--profile=check".to_owned(),
-        ];
+        let mut args = Vec::with_capacity(2);
         if let Some(features) = features {
             args.push("--features".to_owned());
             args.push(features.join(","));
         }
-        args.extend(vec!["--".to_owned(), "-Zunpretty=expanded".to_owned()]);
-
-        let root_content = String::from_utf8(
-            Command::new("cargo")
-                .args(args)
-                .current_dir(root)
-                .stderr(Stdio::inherit())
-                .output()?
-                .stdout,
-        )?;
+        let root_content = expand_program(root.to_path_buf(), package_name, version, None, &args)?;
+        let root_content = String::from_utf8(root_content)?;
 
         let root_file = syn::parse_file(&root_content)?;
         let root_mod = Self::new("crate".to_owned(), root_file.items);