Skip to content

Commit

Permalink
Async Loading outdir and proc-macro
Browse files Browse the repository at this point in the history
  • Loading branch information
@edwin0cheng
edwin0cheng committed Jan 28, 2021
1 parent ad3cb21 commit c01da06
Show file tree
Hide file tree
Showing 11 changed files with 434 additions and 177 deletions.
288 changes: 168 additions & 120 deletions crates/project_model/src/build_data.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,161 +5,207 @@ use std::{
io::BufReader,
path::{Path, PathBuf},
process::{Command, Stdio},
sync::Arc,
};

use anyhow::Result;
use cargo_metadata::{BuildScript, Message, Package, PackageId};
use cargo_metadata::{BuildScript, Message};
use itertools::Itertools;
use paths::{AbsPath, AbsPathBuf};
use rustc_hash::FxHashMap;
use stdx::JodChild;

use crate::{cfg_flag::CfgFlag, CargoConfig};

#[derive(Debug, Clone, Default)]
pub(crate) struct BuildDataMap {
data: FxHashMap<PackageId, BuildData>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BuildData {
pub(crate) struct BuildData {
/// List of config flags defined by this package's build script
pub cfgs: Vec<CfgFlag>,
pub(crate) cfgs: Vec<CfgFlag>,
/// List of cargo-related environment variables with their value
///
/// If the package has a build script which defines environment variables,
/// they can also be found here.
pub envs: Vec<(String, String)>,
pub(crate) envs: Vec<(String, String)>,
/// Directory where a build script might place its output
pub out_dir: Option<AbsPathBuf>,
pub(crate) out_dir: Option<AbsPathBuf>,
/// Path to the proc-macro library file if this package exposes proc-macros
pub proc_macro_dylib_path: Option<AbsPathBuf>,
pub(crate) proc_macro_dylib_path: Option<AbsPathBuf>,
}

impl BuildDataMap {
pub(crate) fn new(
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
packages: &Vec<Package>,
progress: &dyn Fn(String),
) -> Result<BuildDataMap> {
let mut cmd = Command::new(toolchain::cargo());
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
.arg(cargo_toml.as_ref());

// --all-targets includes tests, benches and examples in addition to the
// default lib and bins. This is an independent concept from the --targets
// flag below.
cmd.arg("--all-targets");

if let Some(target) = &cargo_features.target {
cmd.args(&["--target", target]);
}
#[derive(Clone, Debug)]
pub(crate) struct BuildDataConfig {
cargo_toml: AbsPathBuf,
cargo_features: CargoConfig,
packages: Arc<Vec<cargo_metadata::Package>>,
}

if cargo_features.all_features {
cmd.arg("--all-features");
} else {
if cargo_features.no_default_features {
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
// https://github.com/oli-obk/cargo_metadata/issues/79
cmd.arg("--no-default-features");
}
if !cargo_features.features.is_empty() {
cmd.arg("--features");
cmd.arg(cargo_features.features.join(" "));
}
}
impl PartialEq for BuildDataConfig {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.packages, &other.packages)
}
}

cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());

let mut child = cmd.spawn().map(JodChild)?;
let child_stdout = child.stdout.take().unwrap();
let stdout = BufReader::new(child_stdout);

let mut res = BuildDataMap::default();
for message in cargo_metadata::Message::parse_stream(stdout) {
if let Ok(message) = message {
match message {
Message::BuildScriptExecuted(BuildScript {
package_id,
out_dir,
cfgs,
env,
..
}) => {
let cfgs = {
let mut acc = Vec::new();
for cfg in cfgs {
match cfg.parse::<CfgFlag>() {
Ok(it) => acc.push(it),
Err(err) => {
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
}
};
}
acc
};
let res = res.data.entry(package_id.clone()).or_default();
// cargo_metadata crate returns default (empty) path for
// older cargos, which is not absolute, so work around that.
if out_dir != PathBuf::default() {
let out_dir = AbsPathBuf::assert(out_dir);
res.out_dir = Some(out_dir);
res.cfgs = cfgs;
}
impl Eq for BuildDataConfig {}

res.envs = env;
}
Message::CompilerArtifact(message) => {
progress(format!("metadata {}", message.target.name));

if message.target.kind.contains(&"proc-macro".to_string()) {
let package_id = message.package_id;
// Skip rmeta file
if let Some(filename) =
message.filenames.iter().find(|name| is_dylib(name))
{
let filename = AbsPathBuf::assert(filename.clone());
let res = res.data.entry(package_id.clone()).or_default();
res.proc_macro_dylib_path = Some(filename);
}
}
}
Message::CompilerMessage(message) => {
progress(message.target.name.clone());
}
Message::Unknown => (),
Message::BuildFinished(_) => {}
Message::TextLine(_) => {}
}
}
#[derive(Debug, Default)]
pub struct BuildDataCollector {
configs: FxHashMap<AbsPathBuf, BuildDataConfig>,
}

#[derive(Debug, Default)]
pub struct BuildDataResult {
data: FxHashMap<AbsPathBuf, BuildDataMap>,
}

pub(crate) type BuildDataMap = FxHashMap<String, BuildData>;

impl BuildDataCollector {
pub(crate) fn add_config(&mut self, workspace_root: &AbsPath, config: BuildDataConfig) {
self.configs.insert(workspace_root.to_path_buf().clone(), config);
}

pub fn collect(&mut self, progress: &dyn Fn(String)) -> Result<BuildDataResult> {
let mut res = BuildDataResult::default();
for (path, config) in self.configs.iter() {
res.data.insert(
path.clone(),
collect_from_workspace(
&config.cargo_toml,
&config.cargo_features,
&config.packages,
progress,
)?,
);
}
res.inject_cargo_env(packages);
Ok(res)
}
}

impl BuildDataResult {
pub(crate) fn get(&self, root: &AbsPath) -> Option<&BuildDataMap> {
self.data.get(&root.to_path_buf())
}
}

pub(crate) fn with_cargo_env(packages: &Vec<Package>) -> Self {
let mut res = Self::default();
res.inject_cargo_env(packages);
res
impl BuildDataConfig {
pub(crate) fn new(
cargo_toml: AbsPathBuf,
cargo_features: CargoConfig,
packages: Arc<Vec<cargo_metadata::Package>>,
) -> Self {
Self { cargo_toml, cargo_features, packages }
}
}

pub(crate) fn get(&self, id: &PackageId) -> Option<&BuildData> {
self.data.get(id)
fn collect_from_workspace(
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
packages: &Vec<cargo_metadata::Package>,
progress: &dyn Fn(String),
) -> Result<BuildDataMap> {
let mut cmd = Command::new(toolchain::cargo());
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
.arg(cargo_toml.as_ref());

// --all-targets includes tests, benches and examples in addition to the
// default lib and bins. This is an independent concept from the --targets
// flag below.
cmd.arg("--all-targets");

if let Some(target) = &cargo_features.target {
cmd.args(&["--target", target]);
}

fn inject_cargo_env(&mut self, packages: &Vec<Package>) {
for meta_pkg in packages {
let resource = self.data.entry(meta_pkg.id.clone()).or_default();
inject_cargo_env(meta_pkg, &mut resource.envs);
if cargo_features.all_features {
cmd.arg("--all-features");
} else {
if cargo_features.no_default_features {
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
// https://github.com/oli-obk/cargo_metadata/issues/79
cmd.arg("--no-default-features");
}
if !cargo_features.features.is_empty() {
cmd.arg("--features");
cmd.arg(cargo_features.features.join(" "));
}
}

cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());

if let Some(out_dir) = &resource.out_dir {
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
resource.envs.push(("OUT_DIR".to_string(), out_dir));
let mut child = cmd.spawn().map(JodChild)?;
let child_stdout = child.stdout.take().unwrap();
let stdout = BufReader::new(child_stdout);

let mut res = BuildDataMap::default();
for message in cargo_metadata::Message::parse_stream(stdout) {
if let Ok(message) = message {
match message {
Message::BuildScriptExecuted(BuildScript {
package_id,
out_dir,
cfgs,
env,
..
}) => {
let cfgs = {
let mut acc = Vec::new();
for cfg in cfgs {
match cfg.parse::<CfgFlag>() {
Ok(it) => acc.push(it),
Err(err) => {
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
}
};
}
acc
};
let res = res.entry(package_id.repr.clone()).or_default();
// cargo_metadata crate returns default (empty) path for
// older cargos, which is not absolute, so work around that.
if out_dir != PathBuf::default() {
let out_dir = AbsPathBuf::assert(out_dir);
res.out_dir = Some(out_dir);
res.cfgs = cfgs;
}

res.envs = env;
}
Message::CompilerArtifact(message) => {
progress(format!("metadata {}", message.target.name));

if message.target.kind.contains(&"proc-macro".to_string()) {
let package_id = message.package_id;
// Skip rmeta file
if let Some(filename) = message.filenames.iter().find(|name| is_dylib(name))
{
let filename = AbsPathBuf::assert(filename.clone());
let res = res.entry(package_id.repr.clone()).or_default();
res.proc_macro_dylib_path = Some(filename);
}
}
}
Message::CompilerMessage(message) => {
progress(message.target.name.clone());
}
Message::Unknown => (),
Message::BuildFinished(_) => {}
Message::TextLine(_) => {}
}
}
}

for package in packages {
let build_data = res.entry(package.id.repr.clone()).or_default();
inject_cargo_env(package, build_data);
if let Some(out_dir) = &build_data.out_dir {
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
build_data.envs.push(("OUT_DIR".to_string(), out_dir));
}
}
}

Ok(res)
}

// FIXME: File a better way to know if it is a dylib
Expand All @@ -173,7 +219,9 @@ fn is_dylib(path: &Path) -> bool {
/// Recreates the compile-time environment variables that Cargo sets.
///
/// Should be synced with <https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates>
fn inject_cargo_env(package: &cargo_metadata::Package, env: &mut Vec<(String, String)>) {
fn inject_cargo_env(package: &cargo_metadata::Package, build_data: &mut BuildData) {
let env = &mut build_data.envs;

// FIXME: Missing variables:
// CARGO_PKG_HOMEPAGE, CARGO_CRATE_NAME, CARGO_BIN_NAME, CARGO_BIN_EXE_<name>

Expand Down
Loading

0 comments on commit c01da06

Please sign in to comment.