From 9358eecc042d8b551f58d2d5ddb9c88d258880c1 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Thu, 28 Jan 2021 23:33:02 +0800 Subject: Async Loading outdir and proc-macro --- crates/project_model/src/build_data.rs | 288 ++++++++++++++++------------ crates/project_model/src/cargo_workspace.rs | 47 +++-- crates/project_model/src/lib.rs | 1 + crates/project_model/src/workspace.rs | 53 ++++- 4 files changed, 235 insertions(+), 154 deletions(-) (limited to 'crates/project_model') diff --git a/crates/project_model/src/build_data.rs b/crates/project_model/src/build_data.rs index 3ff347e2c..a5c564e0a 100644 --- a/crates/project_model/src/build_data.rs +++ b/crates/project_model/src/build_data.rs @@ -5,10 +5,11 @@ use std::{ io::BufReader, path::{Path, PathBuf}, process::{Command, Stdio}, + sync::Arc, }; use anyhow::Result; -use cargo_metadata::{BuildScript, Message, Package, PackageId}; +use cargo_metadata::{BuildScript, Message}; use itertools::Itertools; use paths::{AbsPath, AbsPathBuf}; use rustc_hash::FxHashMap; @@ -16,150 +17,195 @@ use stdx::JodChild; use crate::{cfg_flag::CfgFlag, CargoConfig}; -#[derive(Debug, Clone, Default)] -pub(crate) struct BuildDataMap { - data: FxHashMap, -} #[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct BuildData { +pub(crate) struct BuildData { /// List of config flags defined by this package's build script - pub cfgs: Vec, + pub(crate) cfgs: Vec, /// List of cargo-related environment variables with their value /// /// If the package has a build script which defines environment variables, /// they can also be found here. - pub envs: Vec<(String, String)>, + pub(crate) envs: Vec<(String, String)>, /// Directory where a build script might place its output - pub out_dir: Option, + pub(crate) out_dir: Option, /// Path to the proc-macro library file if this package exposes proc-macros - pub proc_macro_dylib_path: Option, + pub(crate) proc_macro_dylib_path: Option, } -impl BuildDataMap { - pub(crate) fn new( - cargo_toml: &AbsPath, - cargo_features: &CargoConfig, - packages: &Vec, - progress: &dyn Fn(String), - ) -> Result { - let mut cmd = Command::new(toolchain::cargo()); - cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"]) - .arg(cargo_toml.as_ref()); - - // --all-targets includes tests, benches and examples in addition to the - // default lib and bins. This is an independent concept from the --targets - // flag below. - cmd.arg("--all-targets"); - - if let Some(target) = &cargo_features.target { - cmd.args(&["--target", target]); - } +#[derive(Clone, Debug)] +pub(crate) struct BuildDataConfig { + cargo_toml: AbsPathBuf, + cargo_features: CargoConfig, + packages: Arc>, +} - if cargo_features.all_features { - cmd.arg("--all-features"); - } else { - if cargo_features.no_default_features { - // FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures` - // https://github.com/oli-obk/cargo_metadata/issues/79 - cmd.arg("--no-default-features"); - } - if !cargo_features.features.is_empty() { - cmd.arg("--features"); - cmd.arg(cargo_features.features.join(" ")); - } - } +impl PartialEq for BuildDataConfig { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.packages, &other.packages) + } +} - cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null()); - - let mut child = cmd.spawn().map(JodChild)?; - let child_stdout = child.stdout.take().unwrap(); - let stdout = BufReader::new(child_stdout); - - let mut res = BuildDataMap::default(); - for message in cargo_metadata::Message::parse_stream(stdout) { - if let Ok(message) = message { - match message { - Message::BuildScriptExecuted(BuildScript { - package_id, - out_dir, - cfgs, - env, - .. - }) => { - let cfgs = { - let mut acc = Vec::new(); - for cfg in cfgs { - match cfg.parse::() { - Ok(it) => acc.push(it), - Err(err) => { - anyhow::bail!("invalid cfg from cargo-metadata: {}", err) - } - }; - } - acc - }; - let res = res.data.entry(package_id.clone()).or_default(); - // cargo_metadata crate returns default (empty) path for - // older cargos, which is not absolute, so work around that. - if out_dir != PathBuf::default() { - let out_dir = AbsPathBuf::assert(out_dir); - res.out_dir = Some(out_dir); - res.cfgs = cfgs; - } +impl Eq for BuildDataConfig {} - res.envs = env; - } - Message::CompilerArtifact(message) => { - progress(format!("metadata {}", message.target.name)); - - if message.target.kind.contains(&"proc-macro".to_string()) { - let package_id = message.package_id; - // Skip rmeta file - if let Some(filename) = - message.filenames.iter().find(|name| is_dylib(name)) - { - let filename = AbsPathBuf::assert(filename.clone()); - let res = res.data.entry(package_id.clone()).or_default(); - res.proc_macro_dylib_path = Some(filename); - } - } - } - Message::CompilerMessage(message) => { - progress(message.target.name.clone()); - } - Message::Unknown => (), - Message::BuildFinished(_) => {} - Message::TextLine(_) => {} - } - } +#[derive(Debug, Default)] +pub struct BuildDataCollector { + configs: FxHashMap, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct BuildDataResult { + data: FxHashMap, +} + +pub(crate) type BuildDataMap = FxHashMap; + +impl BuildDataCollector { + pub(crate) fn add_config(&mut self, workspace_root: &AbsPath, config: BuildDataConfig) { + self.configs.insert(workspace_root.to_path_buf().clone(), config); + } + + pub fn collect(&mut self, progress: &dyn Fn(String)) -> Result { + let mut res = BuildDataResult::default(); + for (path, config) in self.configs.iter() { + res.data.insert( + path.clone(), + collect_from_workspace( + &config.cargo_toml, + &config.cargo_features, + &config.packages, + progress, + )?, + ); } - res.inject_cargo_env(packages); Ok(res) } +} + +impl BuildDataResult { + pub(crate) fn get(&self, root: &AbsPath) -> Option<&BuildDataMap> { + self.data.get(&root.to_path_buf()) + } +} - pub(crate) fn with_cargo_env(packages: &Vec) -> Self { - let mut res = Self::default(); - res.inject_cargo_env(packages); - res +impl BuildDataConfig { + pub(crate) fn new( + cargo_toml: AbsPathBuf, + cargo_features: CargoConfig, + packages: Arc>, + ) -> Self { + Self { cargo_toml, cargo_features, packages } } +} - pub(crate) fn get(&self, id: &PackageId) -> Option<&BuildData> { - self.data.get(id) +fn collect_from_workspace( + cargo_toml: &AbsPath, + cargo_features: &CargoConfig, + packages: &Vec, + progress: &dyn Fn(String), +) -> Result { + let mut cmd = Command::new(toolchain::cargo()); + cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"]) + .arg(cargo_toml.as_ref()); + + // --all-targets includes tests, benches and examples in addition to the + // default lib and bins. This is an independent concept from the --targets + // flag below. + cmd.arg("--all-targets"); + + if let Some(target) = &cargo_features.target { + cmd.args(&["--target", target]); } - fn inject_cargo_env(&mut self, packages: &Vec) { - for meta_pkg in packages { - let resource = self.data.entry(meta_pkg.id.clone()).or_default(); - inject_cargo_env(meta_pkg, &mut resource.envs); + if cargo_features.all_features { + cmd.arg("--all-features"); + } else { + if cargo_features.no_default_features { + // FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures` + // https://github.com/oli-obk/cargo_metadata/issues/79 + cmd.arg("--no-default-features"); + } + if !cargo_features.features.is_empty() { + cmd.arg("--features"); + cmd.arg(cargo_features.features.join(" ")); + } + } + + cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null()); - if let Some(out_dir) = &resource.out_dir { - // NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!() - if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) { - resource.envs.push(("OUT_DIR".to_string(), out_dir)); + let mut child = cmd.spawn().map(JodChild)?; + let child_stdout = child.stdout.take().unwrap(); + let stdout = BufReader::new(child_stdout); + + let mut res = BuildDataMap::default(); + for message in cargo_metadata::Message::parse_stream(stdout) { + if let Ok(message) = message { + match message { + Message::BuildScriptExecuted(BuildScript { + package_id, + out_dir, + cfgs, + env, + .. + }) => { + let cfgs = { + let mut acc = Vec::new(); + for cfg in cfgs { + match cfg.parse::() { + Ok(it) => acc.push(it), + Err(err) => { + anyhow::bail!("invalid cfg from cargo-metadata: {}", err) + } + }; + } + acc + }; + let res = res.entry(package_id.repr.clone()).or_default(); + // cargo_metadata crate returns default (empty) path for + // older cargos, which is not absolute, so work around that. + if out_dir != PathBuf::default() { + let out_dir = AbsPathBuf::assert(out_dir); + res.out_dir = Some(out_dir); + res.cfgs = cfgs; + } + + res.envs = env; + } + Message::CompilerArtifact(message) => { + progress(format!("metadata {}", message.target.name)); + + if message.target.kind.contains(&"proc-macro".to_string()) { + let package_id = message.package_id; + // Skip rmeta file + if let Some(filename) = message.filenames.iter().find(|name| is_dylib(name)) + { + let filename = AbsPathBuf::assert(filename.clone()); + let res = res.entry(package_id.repr.clone()).or_default(); + res.proc_macro_dylib_path = Some(filename); + } + } + } + Message::CompilerMessage(message) => { + progress(message.target.name.clone()); } + Message::Unknown => (), + Message::BuildFinished(_) => {} + Message::TextLine(_) => {} } } } + + for package in packages { + let build_data = res.entry(package.id.repr.clone()).or_default(); + inject_cargo_env(package, build_data); + if let Some(out_dir) = &build_data.out_dir { + // NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!() + if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) { + build_data.envs.push(("OUT_DIR".to_string(), out_dir)); + } + } + } + + Ok(res) } // FIXME: File a better way to know if it is a dylib @@ -173,7 +219,9 @@ fn is_dylib(path: &Path) -> bool { /// Recreates the compile-time environment variables that Cargo sets. /// /// Should be synced with -fn inject_cargo_env(package: &cargo_metadata::Package, env: &mut Vec<(String, String)>) { +fn inject_cargo_env(package: &cargo_metadata::Package, build_data: &mut BuildData) { + let env = &mut build_data.envs; + // FIXME: Missing variables: // CARGO_PKG_HOMEPAGE, CARGO_CRATE_NAME, CARGO_BIN_NAME, CARGO_BIN_EXE_ diff --git a/crates/project_model/src/cargo_workspace.rs b/crates/project_model/src/cargo_workspace.rs index c8a5333c4..f47898b9b 100644 --- a/crates/project_model/src/cargo_workspace.rs +++ b/crates/project_model/src/cargo_workspace.rs @@ -1,6 +1,6 @@ //! FIXME: write short doc here -use std::{convert::TryInto, ops, process::Command}; +use std::{convert::TryInto, ops, process::Command, sync::Arc}; use anyhow::{Context, Result}; use base_db::Edition; @@ -9,7 +9,7 @@ use la_arena::{Arena, Idx}; use paths::{AbsPath, AbsPathBuf}; use rustc_hash::FxHashMap; -use crate::build_data::{BuildData, BuildDataMap}; +use crate::build_data::BuildDataConfig; use crate::utf8_stdout; /// `CargoWorkspace` represents the logical structure of, well, a Cargo @@ -27,6 +27,7 @@ pub struct CargoWorkspace { packages: Arena, targets: Arena, workspace_root: AbsPathBuf, + build_data_config: BuildDataConfig, } impl ops::Index for CargoWorkspace { @@ -55,9 +56,6 @@ pub struct CargoConfig { /// This will be ignored if `cargo_all_features` is true. pub features: Vec, - /// Runs cargo check on launch to figure out the correct values of OUT_DIR - pub load_out_dirs_from_check: bool, - /// rustc target pub target: Option, @@ -94,8 +92,8 @@ pub struct PackageData { pub features: FxHashMap>, /// List of features enabled on this package pub active_features: Vec, - /// Build script related data for this package - pub build_data: BuildData, + // String representation of package id + pub id: String, } #[derive(Debug, Clone, Eq, PartialEq)] @@ -228,12 +226,6 @@ impl CargoWorkspace { ) })?; - let resources = if config.load_out_dirs_from_check { - BuildDataMap::new(cargo_toml, config, &meta.packages, progress)? - } else { - BuildDataMap::with_cargo_env(&meta.packages) - }; - let mut pkg_by_id = FxHashMap::default(); let mut packages = Arena::default(); let mut targets = Arena::default(); @@ -241,10 +233,7 @@ impl CargoWorkspace { let ws_members = &meta.workspace_members; meta.packages.sort_by(|a, b| a.id.cmp(&b.id)); - for meta_pkg in meta.packages { - let id = meta_pkg.id.clone(); - let build_data = resources.get(&id).cloned().unwrap_or_default(); - + for meta_pkg in &meta.packages { let cargo_metadata::Package { id, edition, name, manifest_path, version, .. } = meta_pkg; let is_member = ws_members.contains(&id); @@ -252,24 +241,24 @@ impl CargoWorkspace { .parse::() .with_context(|| format!("Failed to parse edition {}", edition))?; let pkg = packages.alloc(PackageData { - name, + id: id.repr.clone(), + name: name.clone(), version: version.to_string(), - manifest: AbsPathBuf::assert(manifest_path), + manifest: AbsPathBuf::assert(manifest_path.clone()), targets: Vec::new(), is_member, edition, dependencies: Vec::new(), - features: meta_pkg.features.into_iter().collect(), + features: meta_pkg.features.clone().into_iter().collect(), active_features: Vec::new(), - build_data, }); let pkg_data = &mut packages[pkg]; pkg_by_id.insert(id, pkg); - for meta_tgt in meta_pkg.targets { + for meta_tgt in &meta_pkg.targets { let is_proc_macro = meta_tgt.kind.as_slice() == ["proc-macro"]; let tgt = targets.alloc(TargetData { package: pkg, - name: meta_tgt.name, + name: meta_tgt.name.clone(), root: AbsPathBuf::assert(meta_tgt.src_path.clone()), kind: TargetKind::new(meta_tgt.kind.as_slice()), is_proc_macro, @@ -308,7 +297,13 @@ impl CargoWorkspace { } let workspace_root = AbsPathBuf::assert(meta.workspace_root); - Ok(CargoWorkspace { packages, targets, workspace_root: workspace_root }) + let build_data_config = BuildDataConfig::new( + cargo_toml.to_path_buf(), + config.clone(), + Arc::new(meta.packages.clone()), + ); + + Ok(CargoWorkspace { packages, targets, workspace_root, build_data_config }) } pub fn packages<'a>(&'a self) -> impl Iterator + ExactSizeIterator + 'a { @@ -334,6 +329,10 @@ impl CargoWorkspace { } } + pub(crate) fn build_data_config(&self) -> &BuildDataConfig { + &self.build_data_config + } + fn is_unique(&self, name: &str) -> bool { self.packages.iter().filter(|(_, v)| v.name == name).count() == 1 } diff --git a/crates/project_model/src/lib.rs b/crates/project_model/src/lib.rs index 525c336e6..d712095a6 100644 --- a/crates/project_model/src/lib.rs +++ b/crates/project_model/src/lib.rs @@ -19,6 +19,7 @@ use paths::{AbsPath, AbsPathBuf}; use rustc_hash::FxHashSet; pub use crate::{ + build_data::{BuildDataCollector, BuildDataResult}, cargo_workspace::{ CargoConfig, CargoWorkspace, Package, PackageData, PackageDependency, Target, TargetData, TargetKind, diff --git a/crates/project_model/src/workspace.rs b/crates/project_model/src/workspace.rs index 559f4e7bf..c30861976 100644 --- a/crates/project_model/src/workspace.rs +++ b/crates/project_model/src/workspace.rs @@ -16,8 +16,13 @@ use proc_macro_api::ProcMacroClient; use rustc_hash::{FxHashMap, FxHashSet}; use crate::{ - cargo_workspace, cfg_flag::CfgFlag, rustc_cfg, sysroot::SysrootCrate, utf8_stdout, CargoConfig, - CargoWorkspace, ProjectJson, ProjectManifest, Sysroot, TargetKind, + build_data::{BuildData, BuildDataMap, BuildDataResult}, + cargo_workspace, + cfg_flag::CfgFlag, + rustc_cfg, + sysroot::SysrootCrate, + utf8_stdout, BuildDataCollector, CargoConfig, CargoWorkspace, ProjectJson, ProjectManifest, + Sysroot, TargetKind, }; /// `PackageRoot` describes a package root folder. @@ -153,7 +158,7 @@ impl ProjectWorkspace { /// Returns the roots for the current `ProjectWorkspace` /// The return type contains the path and whether or not /// the root is a member of the current workspace - pub fn to_roots(&self) -> Vec { + pub fn to_roots(&self, build_data: Option<&BuildDataResult>) -> Vec { match self { ProjectWorkspace::Json { project, sysroot, rustc_cfg: _ } => project .crates() @@ -179,7 +184,12 @@ impl ProjectWorkspace { let pkg_root = cargo[pkg].root().to_path_buf(); let mut include = vec![pkg_root.clone()]; - include.extend(cargo[pkg].build_data.out_dir.clone()); + include.extend( + build_data + .and_then(|it| it.get(cargo.workspace_root())) + .and_then(|map| map.get(&cargo[pkg].id)) + .and_then(|it| it.out_dir.clone()), + ); let mut exclude = vec![pkg_root.join(".git")]; if is_member { @@ -219,6 +229,7 @@ impl ProjectWorkspace { pub fn to_crate_graph( &self, + build_data: Option<&BuildDataResult>, proc_macro_client: Option<&ProcMacroClient>, load: &mut dyn FnMut(&AbsPath) -> Option, ) -> CrateGraph { @@ -241,8 +252,10 @@ impl ProjectWorkspace { &proc_macro_loader, load, cargo, + build_data.and_then(|it| it.get(cargo.workspace_root())), sysroot, rustc, + rustc.as_ref().zip(build_data).and_then(|(it, map)| map.get(it.workspace_root())), ), }; if crate_graph.patch_cfg_if() { @@ -252,6 +265,18 @@ impl ProjectWorkspace { } crate_graph } + + pub fn collect_build_data_configs(&self, collector: &mut BuildDataCollector) { + match self { + ProjectWorkspace::Cargo { cargo, rustc, .. } => { + collector.add_config(&cargo.workspace_root(), cargo.build_data_config().clone()); + if let Some(rustc) = rustc { + collector.add_config(rustc.workspace_root(), rustc.build_data_config().clone()); + } + } + _ => {} + } + } } fn project_json_to_crate_graph( @@ -324,8 +349,10 @@ fn cargo_to_crate_graph( proc_macro_loader: &dyn Fn(&Path) -> Vec, load: &mut dyn FnMut(&AbsPath) -> Option, cargo: &CargoWorkspace, + build_data_map: Option<&BuildDataMap>, sysroot: &Sysroot, rustc: &Option, + rustc_build_data_map: Option<&BuildDataMap>, ) -> CrateGraph { let _p = profile::span("cargo_to_crate_graph"); let mut crate_graph = CrateGraph::default(); @@ -351,6 +378,7 @@ fn cargo_to_crate_graph( let crate_id = add_target_crate_root( &mut crate_graph, &cargo[pkg], + build_data_map.and_then(|it| it.get(&cargo[pkg].id)), &cfg_options, proc_macro_loader, file_id, @@ -427,6 +455,7 @@ fn cargo_to_crate_graph( let crate_id = add_target_crate_root( &mut crate_graph, &rustc_workspace[pkg], + rustc_build_data_map.and_then(|it| it.get(&rustc_workspace[pkg].id)), &cfg_options, proc_macro_loader, file_id, @@ -475,6 +504,7 @@ fn cargo_to_crate_graph( fn add_target_crate_root( crate_graph: &mut CrateGraph, pkg: &cargo_workspace::PackageData, + build_data: Option<&BuildData>, cfg_options: &CfgOptions, proc_macro_loader: &dyn Fn(&Path) -> Vec, file_id: FileId, @@ -485,19 +515,22 @@ fn add_target_crate_root( for feature in pkg.active_features.iter() { opts.insert_key_value("feature".into(), feature.into()); } - opts.extend(pkg.build_data.cfgs.iter().cloned()); + if let Some(cfgs) = build_data.as_ref().map(|it| &it.cfgs) { + opts.extend(cfgs.iter().cloned()); + } opts }; let mut env = Env::default(); - for (k, v) in &pkg.build_data.envs { - env.set(k, v.clone()); + if let Some(envs) = build_data.map(|it| &it.envs) { + for (k, v) in envs { + env.set(k, v.clone()); + } } - let proc_macro = pkg - .build_data - .proc_macro_dylib_path + let proc_macro = build_data .as_ref() + .and_then(|it| it.proc_macro_dylib_path.as_ref()) .map(|it| proc_macro_loader(&it)) .unwrap_or_default(); -- cgit v1.2.3