Skip to content

Commit

Permalink
Async Loading outdir and proc-macro
Browse files Browse the repository at this point in the history
  • Loading branch information
@edwin0cheng
edwin0cheng committed Jan 24, 2021
1 parent 3cd994d commit b114f47
Show file tree
Hide file tree
Showing 10 changed files with 392 additions and 139 deletions.
279 changes: 166 additions & 113 deletions crates/project_model/src/build_data.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,13 @@ use std::{
use anyhow::Result;
use cargo_metadata::{BuildScript, Message, Package, PackageId};
use itertools::Itertools;
use la_arena::{Arena, Idx};
use paths::{AbsPath, AbsPathBuf};
use rustc_hash::FxHashMap;
use stdx::JodChild;

use crate::{cfg_flag::CfgFlag, CargoConfig};

#[derive(Debug, Clone, Default)]
pub(crate) struct BuildDataMap {
data: FxHashMap<PackageId, BuildData>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BuildData {
/// List of config flags defined by this package's build script
Expand All @@ -33,133 +30,187 @@ pub struct BuildData {
pub out_dir: Option<AbsPathBuf>,
/// Path to the proc-macro library file if this package exposes proc-macros
pub proc_macro_dylib_path: Option<AbsPathBuf>,

/// State for build data, used for updating
fetch: Option<PackageId>,
}

impl BuildDataMap {
pub(crate) fn new(
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
packages: &Vec<Package>,
progress: &dyn Fn(String),
) -> Result<BuildDataMap> {
let mut cmd = Command::new(toolchain::cargo());
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
.arg(cargo_toml.as_ref());

// --all-targets includes tests, benches and examples in addition to the
// default lib and bins. This is an independent concept from the --targets
// flag below.
cmd.arg("--all-targets");

if let Some(target) = &cargo_features.target {
cmd.args(&["--target", target]);
}
#[derive(Debug)]
pub(crate) struct BuildDataConfig {
cargo_toml: AbsPathBuf,
cargo_features: CargoConfig,
}

if cargo_features.all_features {
cmd.arg("--all-features");
} else {
if cargo_features.no_default_features {
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
// https://github.com/oli-obk/cargo_metadata/issues/79
cmd.arg("--no-default-features");
}
if !cargo_features.features.is_empty() {
cmd.arg("--features");
cmd.arg(cargo_features.features.join(" "));
}
}
#[derive(Debug, Default)]
pub struct BuildDataLoader {
arena: Arena<BuildDataConfig>,
data: FxHashMap<Idx<BuildDataConfig>, BuildDataMap>,
}

cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());

let mut child = cmd.spawn().map(JodChild)?;
let child_stdout = child.stdout.take().unwrap();
let stdout = BufReader::new(child_stdout);

let mut res = BuildDataMap::default();
for message in cargo_metadata::Message::parse_stream(stdout) {
if let Ok(message) = message {
match message {
Message::BuildScriptExecuted(BuildScript {
package_id,
out_dir,
cfgs,
env,
..
}) => {
let cfgs = {
let mut acc = Vec::new();
for cfg in cfgs {
match cfg.parse::<CfgFlag>() {
Ok(it) => acc.push(it),
Err(err) => {
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
}
};
}
acc
};
let res = res.data.entry(package_id.clone()).or_default();
// cargo_metadata crate returns default (empty) path for
// older cargos, which is not absolute, so work around that.
if out_dir != PathBuf::default() {
let out_dir = AbsPathBuf::assert(out_dir);
res.out_dir = Some(out_dir);
res.cfgs = cfgs;
}
type BuildDataMap = FxHashMap<PackageId, BuildData>;

res.envs = env;
}
Message::CompilerArtifact(message) => {
progress(format!("metadata {}", message.target.name));

if message.target.kind.contains(&"proc-macro".to_string()) {
let package_id = message.package_id;
// Skip rmeta file
if let Some(filename) =
message.filenames.iter().find(|name| is_dylib(name))
{
let filename = AbsPathBuf::assert(filename.clone());
let res = res.data.entry(package_id.clone()).or_default();
res.proc_macro_dylib_path = Some(filename);
}
}
}
Message::CompilerMessage(message) => {
progress(message.target.name.clone());
}
Message::Unknown => (),
Message::BuildFinished(_) => {}
Message::TextLine(_) => {}
pub(crate) fn packages_to_build_data(packages: &Vec<Package>) -> BuildDataMap {
let mut res: BuildDataMap = FxHashMap::default();
for meta_pkg in packages {
let mut build_data = BuildData::default();
build_data.fetch = Some(meta_pkg.id.clone());
inject_cargo_env(meta_pkg, &mut build_data);
res.insert(meta_pkg.id.clone(), build_data);
}
res
}

impl BuildData {
pub(crate) fn update(&mut self, build_data_map: &BuildDataMap) {
if let Some(package_id) = &self.fetch {
let new_data = build_data_map.get(package_id).cloned();
let BuildData { mut cfgs, mut envs, out_dir, proc_macro_dylib_path, .. } =
match new_data {
None => return,
Some(it) => it,
};

if let Some(out_dir) = &out_dir {
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
self.envs.push(("OUT_DIR".to_string(), out_dir));
}
}
self.cfgs.append(&mut cfgs);
self.envs.append(&mut envs);
self.out_dir = out_dir;
self.proc_macro_dylib_path = proc_macro_dylib_path;
self.fetch = None;
}
res.inject_cargo_env(packages);
Ok(res)
}
}

pub(crate) fn with_cargo_env(packages: &Vec<Package>) -> Self {
let mut res = Self::default();
res.inject_cargo_env(packages);
res
impl BuildDataLoader {
pub(crate) fn new_config(
&mut self,
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
) -> Idx<BuildDataConfig> {
self.arena.alloc(BuildDataConfig {
cargo_toml: cargo_toml.to_path_buf().clone(),
cargo_features: cargo_features.clone(),
})
}

pub(crate) fn get(&self, id: &PackageId) -> Option<&BuildData> {
self.data.get(id)
pub fn load(&mut self, progress: &dyn Fn(String)) -> Result<()> {
for (idx, config) in self.arena.iter() {
self.data
.insert(idx, load_workspace(&config.cargo_toml, &config.cargo_features, progress)?);
}
Ok(())
}

fn inject_cargo_env(&mut self, packages: &Vec<Package>) {
for meta_pkg in packages {
let resource = self.data.entry(meta_pkg.id.clone()).or_default();
inject_cargo_env(meta_pkg, &mut resource.envs);
pub fn is_empty(&self) -> bool {
self.arena.is_empty()
}

if let Some(out_dir) = &resource.out_dir {
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
resource.envs.push(("OUT_DIR".to_string(), out_dir));
pub(crate) fn get(&self, idx: &Idx<BuildDataConfig>) -> Option<&BuildDataMap> {
self.data.get(idx)
}
}

fn load_workspace(
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
progress: &dyn Fn(String),
) -> Result<BuildDataMap> {
let mut cmd = Command::new(toolchain::cargo());
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
.arg(cargo_toml.as_ref());

// --all-targets includes tests, benches and examples in addition to the
// default lib and bins. This is an independent concept from the --targets
// flag below.
cmd.arg("--all-targets");

if let Some(target) = &cargo_features.target {
cmd.args(&["--target", target]);
}

if cargo_features.all_features {
cmd.arg("--all-features");
} else {
if cargo_features.no_default_features {
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
// https://github.com/oli-obk/cargo_metadata/issues/79
cmd.arg("--no-default-features");
}
if !cargo_features.features.is_empty() {
cmd.arg("--features");
cmd.arg(cargo_features.features.join(" "));
}
}

cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());

let mut child = cmd.spawn().map(JodChild)?;
let child_stdout = child.stdout.take().unwrap();
let stdout = BufReader::new(child_stdout);

let mut res = FxHashMap::<PackageId, BuildData>::default();
for message in cargo_metadata::Message::parse_stream(stdout) {
if let Ok(message) = message {
match message {
Message::BuildScriptExecuted(BuildScript {
package_id,
out_dir,
cfgs,
env,
..
}) => {
let cfgs = {
let mut acc = Vec::new();
for cfg in cfgs {
match cfg.parse::<CfgFlag>() {
Ok(it) => acc.push(it),
Err(err) => {
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
}
};
}
acc
};
let res = res.entry(package_id.clone()).or_default();
// cargo_metadata crate returns default (empty) path for
// older cargos, which is not absolute, so work around that.
if out_dir != PathBuf::default() {
let out_dir = AbsPathBuf::assert(out_dir);
res.out_dir = Some(out_dir);
res.cfgs = cfgs;
}

res.envs = env;
}
Message::CompilerArtifact(message) => {
progress(format!("metadata {}", message.target.name));

if message.target.kind.contains(&"proc-macro".to_string()) {
let package_id = message.package_id;
// Skip rmeta file
if let Some(filename) = message.filenames.iter().find(|name| is_dylib(name))
{
let filename = AbsPathBuf::assert(filename.clone());
let res = res.entry(package_id.clone()).or_default();
res.proc_macro_dylib_path = Some(filename);
}
}
}
Message::CompilerMessage(message) => {
progress(message.target.name.clone());
}
Message::Unknown => (),
Message::BuildFinished(_) => {}
Message::TextLine(_) => {}
}
}
}

Ok(res)
}

// FIXME: File a better way to know if it is a dylib
Expand All @@ -173,7 +224,9 @@ fn is_dylib(path: &Path) -> bool {
/// Recreates the compile-time environment variables that Cargo sets.
///
/// Should be synced with <https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates>
fn inject_cargo_env(package: &cargo_metadata::Package, env: &mut Vec<(String, String)>) {
fn inject_cargo_env(package: &cargo_metadata::Package, build_data: &mut BuildData) {
let env = &mut build_data.envs;

// FIXME: Missing variables:
// CARGO, CARGO_PKG_HOMEPAGE, CARGO_CRATE_NAME, CARGO_BIN_NAME, CARGO_BIN_EXE_<name>

Expand Down
25 changes: 20 additions & 5 deletions crates/project_model/src/cargo_workspace.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use la_arena::{Arena, Idx};
use paths::{AbsPath, AbsPathBuf};
use rustc_hash::FxHashMap;

use crate::build_data::{BuildData, BuildDataMap};
use crate::build_data::{packages_to_build_data, BuildData, BuildDataConfig, BuildDataLoader};
use crate::utf8_stdout;

/// `CargoWorkspace` represents the logical structure of, well, a Cargo
Expand All @@ -27,6 +27,7 @@ pub struct CargoWorkspace {
packages: Arena<PackageData>,
targets: Arena<TargetData>,
workspace_root: AbsPathBuf,
build_data_config: Option<Idx<BuildDataConfig>>,
}

impl ops::Index<Package> for CargoWorkspace {
Expand Down Expand Up @@ -157,6 +158,7 @@ impl CargoWorkspace {
pub fn from_cargo_metadata(
cargo_toml: &AbsPath,
config: &CargoConfig,
build_data_loader: &mut BuildDataLoader,
progress: &dyn Fn(String),
) -> Result<CargoWorkspace> {
let mut meta = MetadataCommand::new();
Expand Down Expand Up @@ -228,12 +230,13 @@ impl CargoWorkspace {
)
})?;

let resources = if config.load_out_dirs_from_check {
BuildDataMap::new(cargo_toml, config, &meta.packages, progress)?
let build_data_config = if config.load_out_dirs_from_check {
Some(build_data_loader.new_config(cargo_toml, config))
} else {
BuildDataMap::with_cargo_env(&meta.packages)
None
};

let resources = packages_to_build_data(&meta.packages);
let mut pkg_by_id = FxHashMap::default();
let mut packages = Arena::default();
let mut targets = Arena::default();
Expand Down Expand Up @@ -308,7 +311,7 @@ impl CargoWorkspace {
}

let workspace_root = AbsPathBuf::assert(meta.workspace_root);
Ok(CargoWorkspace { packages, targets, workspace_root: workspace_root })
Ok(CargoWorkspace { packages, targets, workspace_root: workspace_root, build_data_config })
}

pub fn packages<'a>(&'a self) -> impl Iterator<Item = Package> + ExactSizeIterator + 'a {
Expand Down Expand Up @@ -337,4 +340,16 @@ impl CargoWorkspace {
fn is_unique(&self, name: &str) -> bool {
self.packages.iter().filter(|(_, v)| v.name == name).count() == 1
}

pub(crate) fn update_build_data(&mut self, loader: &BuildDataLoader) {
let build_data_map = match self.build_data_config.and_then(|it| loader.get(&it)) {
None => return,
Some(it) => it,
};

let keys: Vec<Idx<PackageData>> = self.packages.iter().map(|it| it.0).collect();
for id in keys {
self.packages[id].build_data.update(build_data_map);
}
}
}
Loading

0 comments on commit b114f47

Please sign in to comment.