From 2119dc23e80d77f1abc789e3d99c34d429e17905 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 12:07:28 +0200 Subject: Rename ra_proc_macro -> proc_macro_api --- Cargo.lock | 28 ++-- crates/proc_macro_api/Cargo.toml | 19 +++ crates/proc_macro_api/src/lib.rs | 111 +++++++++++++ crates/proc_macro_api/src/msg.rs | 89 +++++++++++ crates/proc_macro_api/src/process.rs | 201 +++++++++++++++++++++++ crates/proc_macro_api/src/rpc.rs | 267 +++++++++++++++++++++++++++++++ crates/proc_macro_srv/Cargo.toml | 2 +- crates/proc_macro_srv/src/cli.rs | 2 +- crates/proc_macro_srv/src/dylib.rs | 2 +- crates/proc_macro_srv/src/lib.rs | 4 +- crates/proc_macro_srv/src/tests/utils.rs | 2 +- crates/project_model/Cargo.toml | 2 +- crates/project_model/src/lib.rs | 2 +- crates/ra_proc_macro/Cargo.toml | 18 --- crates/ra_proc_macro/src/lib.rs | 111 ------------- crates/ra_proc_macro/src/msg.rs | 89 ----------- crates/ra_proc_macro/src/process.rs | 201 ----------------------- crates/ra_proc_macro/src/rpc.rs | 267 ------------------------------- 18 files changed, 709 insertions(+), 708 deletions(-) create mode 100644 crates/proc_macro_api/Cargo.toml create mode 100644 crates/proc_macro_api/src/lib.rs create mode 100644 crates/proc_macro_api/src/msg.rs create mode 100644 crates/proc_macro_api/src/process.rs create mode 100644 crates/proc_macro_api/src/rpc.rs delete mode 100644 crates/ra_proc_macro/Cargo.toml delete mode 100644 crates/ra_proc_macro/src/lib.rs delete mode 100644 crates/ra_proc_macro/src/msg.rs delete mode 100644 crates/ra_proc_macro/src/process.rs delete mode 100644 crates/ra_proc_macro/src/rpc.rs diff --git a/Cargo.lock b/Cargo.lock index 89d0a60ec..9878dd88f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -909,6 +909,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "proc_macro_api" +version = "0.1.0" +dependencies = [ + "crossbeam-channel", + "jod-thread", + "log", + "serde", + "serde_json", + "tt", +] + [[package]] name = "proc_macro_srv" version = "0.0.0" @@ -919,7 +931,7 @@ dependencies = [ "libloading", "mbe", "memmap", - "ra_proc_macro", + "proc_macro_api", "serde_derive", "test_utils", "toolchain", @@ -948,8 +960,8 @@ dependencies = [ "cfg", "log", "paths", + "proc_macro_api", "ra_db", - "ra_proc_macro", "rustc-hash", "serde", "serde_json", @@ -1131,18 +1143,6 @@ dependencies = [ "text_edit", ] -[[package]] -name = "ra_proc_macro" -version = "0.1.0" -dependencies = [ - "crossbeam-channel", - "jod-thread", - "log", - "serde", - "serde_json", - "tt", -] - [[package]] name = "ra_ssr" version = "0.1.0" diff --git a/crates/proc_macro_api/Cargo.toml b/crates/proc_macro_api/Cargo.toml new file mode 100644 index 000000000..c1abb5627 --- /dev/null +++ b/crates/proc_macro_api/Cargo.toml @@ -0,0 +1,19 @@ +[package] +edition = "2018" +name = "proc_macro_api" +version = "0.1.0" +authors = ["rust-analyzer developers"] +publish = false +license = "MIT OR Apache-2.0" + +[lib] +doctest = false + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +log = "0.4.8" +crossbeam-channel = "0.4.0" +jod-thread = "0.1.1" + +tt = { path = "../tt" } diff --git a/crates/proc_macro_api/src/lib.rs b/crates/proc_macro_api/src/lib.rs new file mode 100644 index 000000000..15db57eb2 --- /dev/null +++ b/crates/proc_macro_api/src/lib.rs @@ -0,0 +1,111 @@ +//! Client-side Proc-Macro crate +//! +//! We separate proc-macro expanding logic to an extern program to allow +//! different implementations (e.g. wasm or dylib loading). And this crate +//! is used to provide basic infrastructure for communication between two +//! processes: Client (RA itself), Server (the external program) + +mod rpc; +mod process; +pub mod msg; + +use std::{ + ffi::OsStr, + io, + path::{Path, PathBuf}, + sync::Arc, +}; + +use tt::{SmolStr, Subtree}; + +use crate::process::{ProcMacroProcessSrv, ProcMacroProcessThread}; + +pub use rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind}; + +#[derive(Debug, Clone)] +pub struct ProcMacroProcessExpander { + process: Arc, + dylib_path: PathBuf, + name: SmolStr, +} + +impl Eq for ProcMacroProcessExpander {} +impl PartialEq for ProcMacroProcessExpander { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.dylib_path == other.dylib_path + && Arc::ptr_eq(&self.process, &other.process) + } +} + +impl tt::TokenExpander for ProcMacroProcessExpander { + fn expand( + &self, + subtree: &Subtree, + _attr: Option<&Subtree>, + ) -> Result { + self.process.custom_derive(&self.dylib_path, subtree, &self.name) + } +} + +#[derive(Debug)] +enum ProcMacroClientKind { + Process { process: Arc, thread: ProcMacroProcessThread }, + Dummy, +} + +#[derive(Debug)] +pub struct ProcMacroClient { + kind: ProcMacroClientKind, +} + +impl ProcMacroClient { + pub fn extern_process( + process_path: PathBuf, + args: impl IntoIterator>, + ) -> io::Result { + let (thread, process) = ProcMacroProcessSrv::run(process_path, args)?; + Ok(ProcMacroClient { + kind: ProcMacroClientKind::Process { process: Arc::new(process), thread }, + }) + } + + pub fn dummy() -> ProcMacroClient { + ProcMacroClient { kind: ProcMacroClientKind::Dummy } + } + + pub fn by_dylib_path(&self, dylib_path: &Path) -> Vec<(SmolStr, Arc)> { + match &self.kind { + ProcMacroClientKind::Dummy => vec![], + ProcMacroClientKind::Process { process, .. } => { + let macros = match process.find_proc_macros(dylib_path) { + Err(err) => { + eprintln!("Failed to find proc macros. Error: {:#?}", err); + return vec![]; + } + Ok(macros) => macros, + }; + + macros + .into_iter() + .filter_map(|(name, kind)| { + // FIXME: Support custom derive only for now. + match kind { + ProcMacroKind::CustomDerive => { + let name = SmolStr::new(&name); + let expander: Arc = + Arc::new(ProcMacroProcessExpander { + process: process.clone(), + name: name.clone(), + dylib_path: dylib_path.into(), + }); + Some((name, expander)) + } + _ => None, + } + }) + .collect() + } + } + } +} diff --git a/crates/proc_macro_api/src/msg.rs b/crates/proc_macro_api/src/msg.rs new file mode 100644 index 000000000..f84ebdbc5 --- /dev/null +++ b/crates/proc_macro_api/src/msg.rs @@ -0,0 +1,89 @@ +//! Defines messages for cross-process message passing based on `ndjson` wire protocol + +use std::{ + convert::TryFrom, + io::{self, BufRead, Write}, +}; + +use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +use crate::{ + rpc::{ListMacrosResult, ListMacrosTask}, + ExpansionResult, ExpansionTask, +}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum Request { + ListMacro(ListMacrosTask), + ExpansionMacro(ExpansionTask), +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum Response { + Error(ResponseError), + ListMacro(ListMacrosResult), + ExpansionMacro(ExpansionResult), +} + +macro_rules! impl_try_from_response { + ($ty:ty, $tag:ident) => { + impl TryFrom for $ty { + type Error = &'static str; + fn try_from(value: Response) -> Result { + match value { + Response::$tag(res) => Ok(res), + _ => Err(concat!("Failed to convert response to ", stringify!($tag))), + } + } + } + }; +} + +impl_try_from_response!(ListMacrosResult, ListMacro); +impl_try_from_response!(ExpansionResult, ExpansionMacro); + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ResponseError { + pub code: ErrorCode, + pub message: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum ErrorCode { + ServerErrorEnd, + ExpansionError, +} + +pub trait Message: Serialize + DeserializeOwned { + fn read(inp: &mut impl BufRead) -> io::Result> { + Ok(match read_json(inp)? { + None => None, + Some(text) => Some(serde_json::from_str(&text)?), + }) + } + fn write(self, out: &mut impl Write) -> io::Result<()> { + let text = serde_json::to_string(&self)?; + write_json(out, &text) + } +} + +impl Message for Request {} +impl Message for Response {} + +fn read_json(inp: &mut impl BufRead) -> io::Result> { + let mut buf = String::new(); + inp.read_line(&mut buf)?; + buf.pop(); // Remove traling '\n' + Ok(match buf.len() { + 0 => None, + _ => Some(buf), + }) +} + +fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> { + log::debug!("> {}", msg); + out.write_all(msg.as_bytes())?; + out.write_all(b"\n")?; + out.flush()?; + Ok(()) +} diff --git a/crates/proc_macro_api/src/process.rs b/crates/proc_macro_api/src/process.rs new file mode 100644 index 000000000..51ffcaa78 --- /dev/null +++ b/crates/proc_macro_api/src/process.rs @@ -0,0 +1,201 @@ +//! Handle process life-time and message passing for proc-macro client + +use std::{ + convert::{TryFrom, TryInto}, + ffi::{OsStr, OsString}, + io::{self, BufRead, BufReader, Write}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::{Arc, Weak}, +}; + +use crossbeam_channel::{bounded, Receiver, Sender}; +use tt::Subtree; + +use crate::{ + msg::{ErrorCode, Message, Request, Response, ResponseError}, + rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind}, +}; + +#[derive(Debug, Default)] +pub(crate) struct ProcMacroProcessSrv { + inner: Option>>, +} + +#[derive(Debug)] +pub(crate) struct ProcMacroProcessThread { + // XXX: drop order is significant + sender: Arc>, + handle: jod_thread::JoinHandle<()>, +} + +impl ProcMacroProcessSrv { + pub fn run( + process_path: PathBuf, + args: impl IntoIterator>, + ) -> io::Result<(ProcMacroProcessThread, ProcMacroProcessSrv)> { + let process = Process::run(process_path, args)?; + + let (task_tx, task_rx) = bounded(0); + let handle = jod_thread::spawn(move || { + client_loop(task_rx, process); + }); + + let task_tx = Arc::new(task_tx); + let srv = ProcMacroProcessSrv { inner: Some(Arc::downgrade(&task_tx)) }; + let thread = ProcMacroProcessThread { handle, sender: task_tx }; + + Ok((thread, srv)) + } + + pub fn find_proc_macros( + &self, + dylib_path: &Path, + ) -> Result, tt::ExpansionError> { + let task = ListMacrosTask { lib: dylib_path.to_path_buf() }; + + let result: ListMacrosResult = self.send_task(Request::ListMacro(task))?; + Ok(result.macros) + } + + pub fn custom_derive( + &self, + dylib_path: &Path, + subtree: &Subtree, + derive_name: &str, + ) -> Result { + let task = ExpansionTask { + macro_body: subtree.clone(), + macro_name: derive_name.to_string(), + attributes: None, + lib: dylib_path.to_path_buf(), + }; + + let result: ExpansionResult = self.send_task(Request::ExpansionMacro(task))?; + Ok(result.expansion) + } + + pub fn send_task(&self, req: Request) -> Result + where + R: TryFrom, + { + let sender = match &self.inner { + None => return Err(tt::ExpansionError::Unknown("No sender is found.".to_string())), + Some(it) => it, + }; + + let (result_tx, result_rx) = bounded(0); + let sender = match sender.upgrade() { + None => { + return Err(tt::ExpansionError::Unknown("Proc macro process is closed.".into())) + } + Some(it) => it, + }; + sender.send(Task { req, result_tx }).unwrap(); + let res = result_rx + .recv() + .map_err(|_| tt::ExpansionError::Unknown("Proc macro thread is closed.".into()))?; + + match res { + Some(Response::Error(err)) => { + return Err(tt::ExpansionError::ExpansionError(err.message)); + } + Some(res) => Ok(res.try_into().map_err(|err| { + tt::ExpansionError::Unknown(format!("Fail to get response, reason : {:#?} ", err)) + })?), + None => Err(tt::ExpansionError::Unknown("Empty result".into())), + } + } +} + +fn client_loop(task_rx: Receiver, mut process: Process) { + let (mut stdin, mut stdout) = match process.stdio() { + None => return, + Some(it) => it, + }; + + for task in task_rx { + let Task { req, result_tx } = task; + + match send_request(&mut stdin, &mut stdout, req) { + Ok(res) => result_tx.send(res).unwrap(), + Err(_err) => { + let res = Response::Error(ResponseError { + code: ErrorCode::ServerErrorEnd, + message: "Server closed".into(), + }); + result_tx.send(res.into()).unwrap(); + // Restart the process + if process.restart().is_err() { + break; + } + let stdio = match process.stdio() { + None => break, + Some(it) => it, + }; + stdin = stdio.0; + stdout = stdio.1; + } + } + } +} + +struct Task { + req: Request, + result_tx: Sender>, +} + +struct Process { + path: PathBuf, + args: Vec, + child: Child, +} + +impl Drop for Process { + fn drop(&mut self) { + let _ = self.child.kill(); + } +} + +impl Process { + fn run( + path: PathBuf, + args: impl IntoIterator>, + ) -> io::Result { + let args = args.into_iter().map(|s| s.as_ref().into()).collect(); + let child = mk_child(&path, &args)?; + Ok(Process { path, args, child }) + } + + fn restart(&mut self) -> io::Result<()> { + let _ = self.child.kill(); + self.child = mk_child(&self.path, &self.args)?; + Ok(()) + } + + fn stdio(&mut self) -> Option<(impl Write, impl BufRead)> { + let stdin = self.child.stdin.take()?; + let stdout = self.child.stdout.take()?; + let read = BufReader::new(stdout); + + Some((stdin, read)) + } +} + +fn mk_child(path: &Path, args: impl IntoIterator>) -> io::Result { + Command::new(&path) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() +} + +fn send_request( + mut writer: &mut impl Write, + mut reader: &mut impl BufRead, + req: Request, +) -> io::Result> { + req.write(&mut writer)?; + Ok(Response::read(&mut reader)?) +} diff --git a/crates/proc_macro_api/src/rpc.rs b/crates/proc_macro_api/src/rpc.rs new file mode 100644 index 000000000..47624163e --- /dev/null +++ b/crates/proc_macro_api/src/rpc.rs @@ -0,0 +1,267 @@ +//! Data structure serialization related stuff for RPC +//! +//! Defines all necessary rpc serialization data structures, +//! which includes `tt` related data and some task messages. +//! Although adding `Serialize` and `Deserialize` traits to `tt` directly seems +//! to be much easier, we deliberately duplicate `tt` structs with `#[serde(with = "XXDef")]` +//! for separation of code responsibility. + +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use tt::{ + Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, SmolStr, Spacing, Subtree, TokenId, + TokenTree, +}; + +#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] +pub struct ListMacrosTask { + pub lib: PathBuf, +} + +#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] +pub enum ProcMacroKind { + CustomDerive, + FuncLike, + Attr, +} + +#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)] +pub struct ListMacrosResult { + pub macros: Vec<(String, ProcMacroKind)>, +} + +#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] +pub struct ExpansionTask { + /// Argument of macro call. + /// + /// In custom derive this will be a struct or enum; in attribute-like macro - underlying + /// item; in function-like macro - the macro body. + #[serde(with = "SubtreeDef")] + pub macro_body: Subtree, + + /// Name of macro to expand. + /// + /// In custom derive this is the name of the derived trait (`Serialize`, `Getters`, etc.). + /// In attribute-like and function-like macros - single name of macro itself (`show_streams`). + pub macro_name: String, + + /// Possible attributes for the attribute-like macros. + #[serde(with = "opt_subtree_def")] + pub attributes: Option, + + pub lib: PathBuf, +} + +#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)] +pub struct ExpansionResult { + #[serde(with = "SubtreeDef")] + pub expansion: Subtree, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "DelimiterKind")] +enum DelimiterKindDef { + Parenthesis, + Brace, + Bracket, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "TokenId")] +struct TokenIdDef(u32); + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Delimiter")] +struct DelimiterDef { + #[serde(with = "TokenIdDef")] + pub id: TokenId, + #[serde(with = "DelimiterKindDef")] + pub kind: DelimiterKind, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Subtree")] +struct SubtreeDef { + #[serde(default, with = "opt_delimiter_def")] + pub delimiter: Option, + #[serde(with = "vec_token_tree")] + pub token_trees: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "TokenTree")] +enum TokenTreeDef { + #[serde(with = "LeafDef")] + Leaf(Leaf), + #[serde(with = "SubtreeDef")] + Subtree(Subtree), +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Leaf")] +enum LeafDef { + #[serde(with = "LiteralDef")] + Literal(Literal), + #[serde(with = "PunctDef")] + Punct(Punct), + #[serde(with = "IdentDef")] + Ident(Ident), +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Literal")] +struct LiteralDef { + pub text: SmolStr, + #[serde(with = "TokenIdDef")] + pub id: TokenId, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Punct")] +struct PunctDef { + pub char: char, + #[serde(with = "SpacingDef")] + pub spacing: Spacing, + #[serde(with = "TokenIdDef")] + pub id: TokenId, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Spacing")] +enum SpacingDef { + Alone, + Joint, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Ident")] +struct IdentDef { + pub text: SmolStr, + #[serde(with = "TokenIdDef")] + pub id: TokenId, +} + +mod opt_delimiter_def { + use super::{Delimiter, DelimiterDef}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(value: &Option, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "DelimiterDef")] &'a Delimiter); + value.as_ref().map(Helper).serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "DelimiterDef")] Delimiter); + let helper = Option::deserialize(deserializer)?; + Ok(helper.map(|Helper(external)| external)) + } +} + +mod opt_subtree_def { + use super::{Subtree, SubtreeDef}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(value: &Option, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "SubtreeDef")] &'a Subtree); + value.as_ref().map(Helper).serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "SubtreeDef")] Subtree); + let helper = Option::deserialize(deserializer)?; + Ok(helper.map(|Helper(external)| external)) + } +} + +mod vec_token_tree { + use super::{TokenTree, TokenTreeDef}; + use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(value: &Vec, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "TokenTreeDef")] &'a TokenTree); + + let items: Vec<_> = value.iter().map(Helper).collect(); + let mut seq = serializer.serialize_seq(Some(items.len()))?; + for element in items { + seq.serialize_element(&element)?; + } + seq.end() + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "TokenTreeDef")] TokenTree); + + let helper = Vec::deserialize(deserializer)?; + Ok(helper.into_iter().map(|Helper(external)| external).collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture_token_tree() -> Subtree { + let mut subtree = Subtree::default(); + subtree + .token_trees + .push(TokenTree::Leaf(Ident { text: "struct".into(), id: TokenId(0) }.into())); + subtree + .token_trees + .push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into())); + subtree.token_trees.push(TokenTree::Subtree( + Subtree { + delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }), + token_trees: vec![], + } + .into(), + )); + subtree + } + + #[test] + fn test_proc_macro_rpc_works() { + let tt = fixture_token_tree(); + let task = ExpansionTask { + macro_body: tt.clone(), + macro_name: Default::default(), + attributes: None, + lib: Default::default(), + }; + + let json = serde_json::to_string(&task).unwrap(); + let back: ExpansionTask = serde_json::from_str(&json).unwrap(); + + assert_eq!(task.macro_body, back.macro_body); + + let result = ExpansionResult { expansion: tt.clone() }; + let json = serde_json::to_string(&result).unwrap(); + let back: ExpansionResult = serde_json::from_str(&json).unwrap(); + + assert_eq!(result, back); + } +} diff --git a/crates/proc_macro_srv/Cargo.toml b/crates/proc_macro_srv/Cargo.toml index 0954ffb66..7171f0808 100644 --- a/crates/proc_macro_srv/Cargo.toml +++ b/crates/proc_macro_srv/Cargo.toml @@ -15,7 +15,7 @@ memmap = "0.7" tt = { path = "../tt" } mbe = { path = "../mbe" } -ra_proc_macro = { path = "../ra_proc_macro" } +proc_macro_api = { path = "../proc_macro_api" } test_utils = { path = "../test_utils" } [dev-dependencies] diff --git a/crates/proc_macro_srv/src/cli.rs b/crates/proc_macro_srv/src/cli.rs index 1437794c9..d428b9567 100644 --- a/crates/proc_macro_srv/src/cli.rs +++ b/crates/proc_macro_srv/src/cli.rs @@ -1,7 +1,7 @@ //! Driver for proc macro server use crate::ProcMacroSrv; -use ra_proc_macro::msg::{self, Message}; +use proc_macro_api::msg::{self, Message}; use std::io; pub fn run() -> io::Result<()> { diff --git a/crates/proc_macro_srv/src/dylib.rs b/crates/proc_macro_srv/src/dylib.rs index 9b6cc91ef..f8f705da8 100644 --- a/crates/proc_macro_srv/src/dylib.rs +++ b/crates/proc_macro_srv/src/dylib.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; use goblin::{mach::Mach, Object}; use libloading::Library; use memmap::Mmap; -use ra_proc_macro::ProcMacroKind; +use proc_macro_api::ProcMacroKind; use std::io; const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_"; diff --git a/crates/proc_macro_srv/src/lib.rs b/crates/proc_macro_srv/src/lib.rs index 1fc2eef82..7e4e4ad50 100644 --- a/crates/proc_macro_srv/src/lib.rs +++ b/crates/proc_macro_srv/src/lib.rs @@ -8,7 +8,7 @@ //! * We use `tt` for proc-macro `TokenStream` server, it is easier to manipulate and interact with //! RA than `proc-macro2` token stream. //! * By **copying** the whole rustc `lib_proc_macro` code, we are able to build this with `stable` -//! rustc rather than `unstable`. (Although in gerenal ABI compatibility is still an issue) +//! rustc rather than `unstable`. (Although in general ABI compatibility is still an issue)… #[allow(dead_code)] #[doc(hidden)] @@ -20,7 +20,7 @@ mod rustc_server; mod dylib; use proc_macro::bridge::client::TokenStream; -use ra_proc_macro::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask}; +use proc_macro_api::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask}; use std::{ collections::{hash_map::Entry, HashMap}, fs, diff --git a/crates/proc_macro_srv/src/tests/utils.rs b/crates/proc_macro_srv/src/tests/utils.rs index 1b6a0b6fb..5828512d6 100644 --- a/crates/proc_macro_srv/src/tests/utils.rs +++ b/crates/proc_macro_srv/src/tests/utils.rs @@ -2,7 +2,7 @@ use crate::dylib; use crate::ProcMacroSrv; -use ra_proc_macro::ListMacrosTask; +use proc_macro_api::ListMacrosTask; use std::str::FromStr; use test_utils::assert_eq_text; diff --git a/crates/project_model/Cargo.toml b/crates/project_model/Cargo.toml index 8d8d09387..1c84c7d20 100644 --- a/crates/project_model/Cargo.toml +++ b/crates/project_model/Cargo.toml @@ -20,6 +20,6 @@ arena = { path = "../arena" } cfg = { path = "../cfg" } ra_db = { path = "../ra_db" } toolchain = { path = "../toolchain" } -ra_proc_macro = { path = "../ra_proc_macro" } +proc_macro_api = { path = "../proc_macro_api" } paths = { path = "../paths" } stdx = { path = "../stdx" } diff --git a/crates/project_model/src/lib.rs b/crates/project_model/src/lib.rs index ee42198f3..234f908c9 100644 --- a/crates/project_model/src/lib.rs +++ b/crates/project_model/src/lib.rs @@ -25,7 +25,7 @@ pub use crate::{ sysroot::Sysroot, }; -pub use ra_proc_macro::ProcMacroClient; +pub use proc_macro_api::ProcMacroClient; #[derive(Debug, Clone, Eq, PartialEq)] pub enum ProjectWorkspace { diff --git a/crates/ra_proc_macro/Cargo.toml b/crates/ra_proc_macro/Cargo.toml deleted file mode 100644 index d2d1bc228..000000000 --- a/crates/ra_proc_macro/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -edition = "2018" -name = "ra_proc_macro" -version = "0.1.0" -authors = ["rust-analyzer developers"] -publish = false -license = "MIT OR Apache-2.0" - -[lib] -doctest = false - -[dependencies] -tt = { path = "../tt" } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -log = "0.4.8" -crossbeam-channel = "0.4.0" -jod-thread = "0.1.1" diff --git a/crates/ra_proc_macro/src/lib.rs b/crates/ra_proc_macro/src/lib.rs deleted file mode 100644 index 15db57eb2..000000000 --- a/crates/ra_proc_macro/src/lib.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Client-side Proc-Macro crate -//! -//! We separate proc-macro expanding logic to an extern program to allow -//! different implementations (e.g. wasm or dylib loading). And this crate -//! is used to provide basic infrastructure for communication between two -//! processes: Client (RA itself), Server (the external program) - -mod rpc; -mod process; -pub mod msg; - -use std::{ - ffi::OsStr, - io, - path::{Path, PathBuf}, - sync::Arc, -}; - -use tt::{SmolStr, Subtree}; - -use crate::process::{ProcMacroProcessSrv, ProcMacroProcessThread}; - -pub use rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind}; - -#[derive(Debug, Clone)] -pub struct ProcMacroProcessExpander { - process: Arc, - dylib_path: PathBuf, - name: SmolStr, -} - -impl Eq for ProcMacroProcessExpander {} -impl PartialEq for ProcMacroProcessExpander { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - && self.dylib_path == other.dylib_path - && Arc::ptr_eq(&self.process, &other.process) - } -} - -impl tt::TokenExpander for ProcMacroProcessExpander { - fn expand( - &self, - subtree: &Subtree, - _attr: Option<&Subtree>, - ) -> Result { - self.process.custom_derive(&self.dylib_path, subtree, &self.name) - } -} - -#[derive(Debug)] -enum ProcMacroClientKind { - Process { process: Arc, thread: ProcMacroProcessThread }, - Dummy, -} - -#[derive(Debug)] -pub struct ProcMacroClient { - kind: ProcMacroClientKind, -} - -impl ProcMacroClient { - pub fn extern_process( - process_path: PathBuf, - args: impl IntoIterator>, - ) -> io::Result { - let (thread, process) = ProcMacroProcessSrv::run(process_path, args)?; - Ok(ProcMacroClient { - kind: ProcMacroClientKind::Process { process: Arc::new(process), thread }, - }) - } - - pub fn dummy() -> ProcMacroClient { - ProcMacroClient { kind: ProcMacroClientKind::Dummy } - } - - pub fn by_dylib_path(&self, dylib_path: &Path) -> Vec<(SmolStr, Arc)> { - match &self.kind { - ProcMacroClientKind::Dummy => vec![], - ProcMacroClientKind::Process { process, .. } => { - let macros = match process.find_proc_macros(dylib_path) { - Err(err) => { - eprintln!("Failed to find proc macros. Error: {:#?}", err); - return vec![]; - } - Ok(macros) => macros, - }; - - macros - .into_iter() - .filter_map(|(name, kind)| { - // FIXME: Support custom derive only for now. - match kind { - ProcMacroKind::CustomDerive => { - let name = SmolStr::new(&name); - let expander: Arc = - Arc::new(ProcMacroProcessExpander { - process: process.clone(), - name: name.clone(), - dylib_path: dylib_path.into(), - }); - Some((name, expander)) - } - _ => None, - } - }) - .collect() - } - } - } -} diff --git a/crates/ra_proc_macro/src/msg.rs b/crates/ra_proc_macro/src/msg.rs deleted file mode 100644 index f84ebdbc5..000000000 --- a/crates/ra_proc_macro/src/msg.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! Defines messages for cross-process message passing based on `ndjson` wire protocol - -use std::{ - convert::TryFrom, - io::{self, BufRead, Write}, -}; - -use serde::{de::DeserializeOwned, Deserialize, Serialize}; - -use crate::{ - rpc::{ListMacrosResult, ListMacrosTask}, - ExpansionResult, ExpansionTask, -}; - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub enum Request { - ListMacro(ListMacrosTask), - ExpansionMacro(ExpansionTask), -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub enum Response { - Error(ResponseError), - ListMacro(ListMacrosResult), - ExpansionMacro(ExpansionResult), -} - -macro_rules! impl_try_from_response { - ($ty:ty, $tag:ident) => { - impl TryFrom for $ty { - type Error = &'static str; - fn try_from(value: Response) -> Result { - match value { - Response::$tag(res) => Ok(res), - _ => Err(concat!("Failed to convert response to ", stringify!($tag))), - } - } - } - }; -} - -impl_try_from_response!(ListMacrosResult, ListMacro); -impl_try_from_response!(ExpansionResult, ExpansionMacro); - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct ResponseError { - pub code: ErrorCode, - pub message: String, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub enum ErrorCode { - ServerErrorEnd, - ExpansionError, -} - -pub trait Message: Serialize + DeserializeOwned { - fn read(inp: &mut impl BufRead) -> io::Result> { - Ok(match read_json(inp)? { - None => None, - Some(text) => Some(serde_json::from_str(&text)?), - }) - } - fn write(self, out: &mut impl Write) -> io::Result<()> { - let text = serde_json::to_string(&self)?; - write_json(out, &text) - } -} - -impl Message for Request {} -impl Message for Response {} - -fn read_json(inp: &mut impl BufRead) -> io::Result> { - let mut buf = String::new(); - inp.read_line(&mut buf)?; - buf.pop(); // Remove traling '\n' - Ok(match buf.len() { - 0 => None, - _ => Some(buf), - }) -} - -fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> { - log::debug!("> {}", msg); - out.write_all(msg.as_bytes())?; - out.write_all(b"\n")?; - out.flush()?; - Ok(()) -} diff --git a/crates/ra_proc_macro/src/process.rs b/crates/ra_proc_macro/src/process.rs deleted file mode 100644 index 51ffcaa78..000000000 --- a/crates/ra_proc_macro/src/process.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! Handle process life-time and message passing for proc-macro client - -use std::{ - convert::{TryFrom, TryInto}, - ffi::{OsStr, OsString}, - io::{self, BufRead, BufReader, Write}, - path::{Path, PathBuf}, - process::{Child, Command, Stdio}, - sync::{Arc, Weak}, -}; - -use crossbeam_channel::{bounded, Receiver, Sender}; -use tt::Subtree; - -use crate::{ - msg::{ErrorCode, Message, Request, Response, ResponseError}, - rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind}, -}; - -#[derive(Debug, Default)] -pub(crate) struct ProcMacroProcessSrv { - inner: Option>>, -} - -#[derive(Debug)] -pub(crate) struct ProcMacroProcessThread { - // XXX: drop order is significant - sender: Arc>, - handle: jod_thread::JoinHandle<()>, -} - -impl ProcMacroProcessSrv { - pub fn run( - process_path: PathBuf, - args: impl IntoIterator>, - ) -> io::Result<(ProcMacroProcessThread, ProcMacroProcessSrv)> { - let process = Process::run(process_path, args)?; - - let (task_tx, task_rx) = bounded(0); - let handle = jod_thread::spawn(move || { - client_loop(task_rx, process); - }); - - let task_tx = Arc::new(task_tx); - let srv = ProcMacroProcessSrv { inner: Some(Arc::downgrade(&task_tx)) }; - let thread = ProcMacroProcessThread { handle, sender: task_tx }; - - Ok((thread, srv)) - } - - pub fn find_proc_macros( - &self, - dylib_path: &Path, - ) -> Result, tt::ExpansionError> { - let task = ListMacrosTask { lib: dylib_path.to_path_buf() }; - - let result: ListMacrosResult = self.send_task(Request::ListMacro(task))?; - Ok(result.macros) - } - - pub fn custom_derive( - &self, - dylib_path: &Path, - subtree: &Subtree, - derive_name: &str, - ) -> Result { - let task = ExpansionTask { - macro_body: subtree.clone(), - macro_name: derive_name.to_string(), - attributes: None, - lib: dylib_path.to_path_buf(), - }; - - let result: ExpansionResult = self.send_task(Request::ExpansionMacro(task))?; - Ok(result.expansion) - } - - pub fn send_task(&self, req: Request) -> Result - where - R: TryFrom, - { - let sender = match &self.inner { - None => return Err(tt::ExpansionError::Unknown("No sender is found.".to_string())), - Some(it) => it, - }; - - let (result_tx, result_rx) = bounded(0); - let sender = match sender.upgrade() { - None => { - return Err(tt::ExpansionError::Unknown("Proc macro process is closed.".into())) - } - Some(it) => it, - }; - sender.send(Task { req, result_tx }).unwrap(); - let res = result_rx - .recv() - .map_err(|_| tt::ExpansionError::Unknown("Proc macro thread is closed.".into()))?; - - match res { - Some(Response::Error(err)) => { - return Err(tt::ExpansionError::ExpansionError(err.message)); - } - Some(res) => Ok(res.try_into().map_err(|err| { - tt::ExpansionError::Unknown(format!("Fail to get response, reason : {:#?} ", err)) - })?), - None => Err(tt::ExpansionError::Unknown("Empty result".into())), - } - } -} - -fn client_loop(task_rx: Receiver, mut process: Process) { - let (mut stdin, mut stdout) = match process.stdio() { - None => return, - Some(it) => it, - }; - - for task in task_rx { - let Task { req, result_tx } = task; - - match send_request(&mut stdin, &mut stdout, req) { - Ok(res) => result_tx.send(res).unwrap(), - Err(_err) => { - let res = Response::Error(ResponseError { - code: ErrorCode::ServerErrorEnd, - message: "Server closed".into(), - }); - result_tx.send(res.into()).unwrap(); - // Restart the process - if process.restart().is_err() { - break; - } - let stdio = match process.stdio() { - None => break, - Some(it) => it, - }; - stdin = stdio.0; - stdout = stdio.1; - } - } - } -} - -struct Task { - req: Request, - result_tx: Sender>, -} - -struct Process { - path: PathBuf, - args: Vec, - child: Child, -} - -impl Drop for Process { - fn drop(&mut self) { - let _ = self.child.kill(); - } -} - -impl Process { - fn run( - path: PathBuf, - args: impl IntoIterator>, - ) -> io::Result { - let args = args.into_iter().map(|s| s.as_ref().into()).collect(); - let child = mk_child(&path, &args)?; - Ok(Process { path, args, child }) - } - - fn restart(&mut self) -> io::Result<()> { - let _ = self.child.kill(); - self.child = mk_child(&self.path, &self.args)?; - Ok(()) - } - - fn stdio(&mut self) -> Option<(impl Write, impl BufRead)> { - let stdin = self.child.stdin.take()?; - let stdout = self.child.stdout.take()?; - let read = BufReader::new(stdout); - - Some((stdin, read)) - } -} - -fn mk_child(path: &Path, args: impl IntoIterator>) -> io::Result { - Command::new(&path) - .args(args) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()) - .spawn() -} - -fn send_request( - mut writer: &mut impl Write, - mut reader: &mut impl BufRead, - req: Request, -) -> io::Result> { - req.write(&mut writer)?; - Ok(Response::read(&mut reader)?) -} diff --git a/crates/ra_proc_macro/src/rpc.rs b/crates/ra_proc_macro/src/rpc.rs deleted file mode 100644 index 5e5d78d06..000000000 --- a/crates/ra_proc_macro/src/rpc.rs +++ /dev/null @@ -1,267 +0,0 @@ -//! Data struture serialization related stuff for RPC -//! -//! Defines all necessary rpc serialization data structures, -//! which includes `tt` related data and some task messages. -//! Although adding `Serialize` and `Deserialize` traits to `tt` directly seems -//! to be much easier, we deliberately duplicate `tt` structs with `#[serde(with = "XXDef")]` -//! for separation of code responsibility. - -use std::path::PathBuf; - -use serde::{Deserialize, Serialize}; -use tt::{ - Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, SmolStr, Spacing, Subtree, TokenId, - TokenTree, -}; - -#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] -pub struct ListMacrosTask { - pub lib: PathBuf, -} - -#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] -pub enum ProcMacroKind { - CustomDerive, - FuncLike, - Attr, -} - -#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)] -pub struct ListMacrosResult { - pub macros: Vec<(String, ProcMacroKind)>, -} - -#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] -pub struct ExpansionTask { - /// Argument of macro call. - /// - /// In custom derive this will be a struct or enum; in attribute-like macro - underlying - /// item; in function-like macro - the macro body. - #[serde(with = "SubtreeDef")] - pub macro_body: Subtree, - - /// Name of macro to expand. - /// - /// In custom derive this is the name of the derived trait (`Serialize`, `Getters`, etc.). - /// In attribute-like and function-like macros - single name of macro itself (`show_streams`). - pub macro_name: String, - - /// Possible attributes for the attribute-like macros. - #[serde(with = "opt_subtree_def")] - pub attributes: Option, - - pub lib: PathBuf, -} - -#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)] -pub struct ExpansionResult { - #[serde(with = "SubtreeDef")] - pub expansion: Subtree, -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "DelimiterKind")] -enum DelimiterKindDef { - Parenthesis, - Brace, - Bracket, -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "TokenId")] -struct TokenIdDef(u32); - -#[derive(Serialize, Deserialize)] -#[serde(remote = "Delimiter")] -struct DelimiterDef { - #[serde(with = "TokenIdDef")] - pub id: TokenId, - #[serde(with = "DelimiterKindDef")] - pub kind: DelimiterKind, -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "Subtree")] -struct SubtreeDef { - #[serde(default, with = "opt_delimiter_def")] - pub delimiter: Option, - #[serde(with = "vec_token_tree")] - pub token_trees: Vec, -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "TokenTree")] -enum TokenTreeDef { - #[serde(with = "LeafDef")] - Leaf(Leaf), - #[serde(with = "SubtreeDef")] - Subtree(Subtree), -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "Leaf")] -enum LeafDef { - #[serde(with = "LiteralDef")] - Literal(Literal), - #[serde(with = "PunctDef")] - Punct(Punct), - #[serde(with = "IdentDef")] - Ident(Ident), -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "Literal")] -struct LiteralDef { - pub text: SmolStr, - #[serde(with = "TokenIdDef")] - pub id: TokenId, -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "Punct")] -struct PunctDef { - pub char: char, - #[serde(with = "SpacingDef")] - pub spacing: Spacing, - #[serde(with = "TokenIdDef")] - pub id: TokenId, -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "Spacing")] -enum SpacingDef { - Alone, - Joint, -} - -#[derive(Serialize, Deserialize)] -#[serde(remote = "Ident")] -struct IdentDef { - pub text: SmolStr, - #[serde(with = "TokenIdDef")] - pub id: TokenId, -} - -mod opt_delimiter_def { - use super::{Delimiter, DelimiterDef}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - pub fn serialize(value: &Option, serializer: S) -> Result - where - S: Serializer, - { - #[derive(Serialize)] - struct Helper<'a>(#[serde(with = "DelimiterDef")] &'a Delimiter); - value.as_ref().map(Helper).serialize(serializer) - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - struct Helper(#[serde(with = "DelimiterDef")] Delimiter); - let helper = Option::deserialize(deserializer)?; - Ok(helper.map(|Helper(external)| external)) - } -} - -mod opt_subtree_def { - use super::{Subtree, SubtreeDef}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - pub fn serialize(value: &Option, serializer: S) -> Result - where - S: Serializer, - { - #[derive(Serialize)] - struct Helper<'a>(#[serde(with = "SubtreeDef")] &'a Subtree); - value.as_ref().map(Helper).serialize(serializer) - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - struct Helper(#[serde(with = "SubtreeDef")] Subtree); - let helper = Option::deserialize(deserializer)?; - Ok(helper.map(|Helper(external)| external)) - } -} - -mod vec_token_tree { - use super::{TokenTree, TokenTreeDef}; - use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer}; - - pub fn serialize(value: &Vec, serializer: S) -> Result - where - S: Serializer, - { - #[derive(Serialize)] - struct Helper<'a>(#[serde(with = "TokenTreeDef")] &'a TokenTree); - - let items: Vec<_> = value.iter().map(Helper).collect(); - let mut seq = serializer.serialize_seq(Some(items.len()))?; - for element in items { - seq.serialize_element(&element)?; - } - seq.end() - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - struct Helper(#[serde(with = "TokenTreeDef")] TokenTree); - - let helper = Vec::deserialize(deserializer)?; - Ok(helper.into_iter().map(|Helper(external)| external).collect()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn fixture_token_tree() -> Subtree { - let mut subtree = Subtree::default(); - subtree - .token_trees - .push(TokenTree::Leaf(Ident { text: "struct".into(), id: TokenId(0) }.into())); - subtree - .token_trees - .push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into())); - subtree.token_trees.push(TokenTree::Subtree( - Subtree { - delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }), - token_trees: vec![], - } - .into(), - )); - subtree - } - - #[test] - fn test_proc_macro_rpc_works() { - let tt = fixture_token_tree(); - let task = ExpansionTask { - macro_body: tt.clone(), - macro_name: Default::default(), - attributes: None, - lib: Default::default(), - }; - - let json = serde_json::to_string(&task).unwrap(); - let back: ExpansionTask = serde_json::from_str(&json).unwrap(); - - assert_eq!(task.macro_body, back.macro_body); - - let result = ExpansionResult { expansion: tt.clone() }; - let json = serde_json::to_string(&result).unwrap(); - let back: ExpansionResult = serde_json::from_str(&json).unwrap(); - - assert_eq!(result, back); - } -} -- cgit v1.2.3