From 2119dc23e80d77f1abc789e3d99c34d429e17905 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 12:07:28 +0200 Subject: Rename ra_proc_macro -> proc_macro_api --- crates/proc_macro_api/Cargo.toml | 19 +++ crates/proc_macro_api/src/lib.rs | 111 +++++++++++++++ crates/proc_macro_api/src/msg.rs | 89 ++++++++++++ crates/proc_macro_api/src/process.rs | 201 ++++++++++++++++++++++++++ crates/proc_macro_api/src/rpc.rs | 267 +++++++++++++++++++++++++++++++++++ 5 files changed, 687 insertions(+) create mode 100644 crates/proc_macro_api/Cargo.toml create mode 100644 crates/proc_macro_api/src/lib.rs create mode 100644 crates/proc_macro_api/src/msg.rs create mode 100644 crates/proc_macro_api/src/process.rs create mode 100644 crates/proc_macro_api/src/rpc.rs (limited to 'crates/proc_macro_api') diff --git a/crates/proc_macro_api/Cargo.toml b/crates/proc_macro_api/Cargo.toml new file mode 100644 index 000000000..c1abb5627 --- /dev/null +++ b/crates/proc_macro_api/Cargo.toml @@ -0,0 +1,19 @@ +[package] +edition = "2018" +name = "proc_macro_api" +version = "0.1.0" +authors = ["rust-analyzer developers"] +publish = false +license = "MIT OR Apache-2.0" + +[lib] +doctest = false + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +log = "0.4.8" +crossbeam-channel = "0.4.0" +jod-thread = "0.1.1" + +tt = { path = "../tt" } diff --git a/crates/proc_macro_api/src/lib.rs b/crates/proc_macro_api/src/lib.rs new file mode 100644 index 000000000..15db57eb2 --- /dev/null +++ b/crates/proc_macro_api/src/lib.rs @@ -0,0 +1,111 @@ +//! Client-side Proc-Macro crate +//! +//! We separate proc-macro expanding logic to an extern program to allow +//! different implementations (e.g. wasm or dylib loading). And this crate +//! is used to provide basic infrastructure for communication between two +//! processes: Client (RA itself), Server (the external program) + +mod rpc; +mod process; +pub mod msg; + +use std::{ + ffi::OsStr, + io, + path::{Path, PathBuf}, + sync::Arc, +}; + +use tt::{SmolStr, Subtree}; + +use crate::process::{ProcMacroProcessSrv, ProcMacroProcessThread}; + +pub use rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind}; + +#[derive(Debug, Clone)] +pub struct ProcMacroProcessExpander { + process: Arc, + dylib_path: PathBuf, + name: SmolStr, +} + +impl Eq for ProcMacroProcessExpander {} +impl PartialEq for ProcMacroProcessExpander { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.dylib_path == other.dylib_path + && Arc::ptr_eq(&self.process, &other.process) + } +} + +impl tt::TokenExpander for ProcMacroProcessExpander { + fn expand( + &self, + subtree: &Subtree, + _attr: Option<&Subtree>, + ) -> Result { + self.process.custom_derive(&self.dylib_path, subtree, &self.name) + } +} + +#[derive(Debug)] +enum ProcMacroClientKind { + Process { process: Arc, thread: ProcMacroProcessThread }, + Dummy, +} + +#[derive(Debug)] +pub struct ProcMacroClient { + kind: ProcMacroClientKind, +} + +impl ProcMacroClient { + pub fn extern_process( + process_path: PathBuf, + args: impl IntoIterator>, + ) -> io::Result { + let (thread, process) = ProcMacroProcessSrv::run(process_path, args)?; + Ok(ProcMacroClient { + kind: ProcMacroClientKind::Process { process: Arc::new(process), thread }, + }) + } + + pub fn dummy() -> ProcMacroClient { + ProcMacroClient { kind: ProcMacroClientKind::Dummy } + } + + pub fn by_dylib_path(&self, dylib_path: &Path) -> Vec<(SmolStr, Arc)> { + match &self.kind { + ProcMacroClientKind::Dummy => vec![], + ProcMacroClientKind::Process { process, .. } => { + let macros = match process.find_proc_macros(dylib_path) { + Err(err) => { + eprintln!("Failed to find proc macros. Error: {:#?}", err); + return vec![]; + } + Ok(macros) => macros, + }; + + macros + .into_iter() + .filter_map(|(name, kind)| { + // FIXME: Support custom derive only for now. + match kind { + ProcMacroKind::CustomDerive => { + let name = SmolStr::new(&name); + let expander: Arc = + Arc::new(ProcMacroProcessExpander { + process: process.clone(), + name: name.clone(), + dylib_path: dylib_path.into(), + }); + Some((name, expander)) + } + _ => None, + } + }) + .collect() + } + } + } +} diff --git a/crates/proc_macro_api/src/msg.rs b/crates/proc_macro_api/src/msg.rs new file mode 100644 index 000000000..f84ebdbc5 --- /dev/null +++ b/crates/proc_macro_api/src/msg.rs @@ -0,0 +1,89 @@ +//! Defines messages for cross-process message passing based on `ndjson` wire protocol + +use std::{ + convert::TryFrom, + io::{self, BufRead, Write}, +}; + +use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +use crate::{ + rpc::{ListMacrosResult, ListMacrosTask}, + ExpansionResult, ExpansionTask, +}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum Request { + ListMacro(ListMacrosTask), + ExpansionMacro(ExpansionTask), +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum Response { + Error(ResponseError), + ListMacro(ListMacrosResult), + ExpansionMacro(ExpansionResult), +} + +macro_rules! impl_try_from_response { + ($ty:ty, $tag:ident) => { + impl TryFrom for $ty { + type Error = &'static str; + fn try_from(value: Response) -> Result { + match value { + Response::$tag(res) => Ok(res), + _ => Err(concat!("Failed to convert response to ", stringify!($tag))), + } + } + } + }; +} + +impl_try_from_response!(ListMacrosResult, ListMacro); +impl_try_from_response!(ExpansionResult, ExpansionMacro); + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ResponseError { + pub code: ErrorCode, + pub message: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum ErrorCode { + ServerErrorEnd, + ExpansionError, +} + +pub trait Message: Serialize + DeserializeOwned { + fn read(inp: &mut impl BufRead) -> io::Result> { + Ok(match read_json(inp)? { + None => None, + Some(text) => Some(serde_json::from_str(&text)?), + }) + } + fn write(self, out: &mut impl Write) -> io::Result<()> { + let text = serde_json::to_string(&self)?; + write_json(out, &text) + } +} + +impl Message for Request {} +impl Message for Response {} + +fn read_json(inp: &mut impl BufRead) -> io::Result> { + let mut buf = String::new(); + inp.read_line(&mut buf)?; + buf.pop(); // Remove traling '\n' + Ok(match buf.len() { + 0 => None, + _ => Some(buf), + }) +} + +fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> { + log::debug!("> {}", msg); + out.write_all(msg.as_bytes())?; + out.write_all(b"\n")?; + out.flush()?; + Ok(()) +} diff --git a/crates/proc_macro_api/src/process.rs b/crates/proc_macro_api/src/process.rs new file mode 100644 index 000000000..51ffcaa78 --- /dev/null +++ b/crates/proc_macro_api/src/process.rs @@ -0,0 +1,201 @@ +//! Handle process life-time and message passing for proc-macro client + +use std::{ + convert::{TryFrom, TryInto}, + ffi::{OsStr, OsString}, + io::{self, BufRead, BufReader, Write}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::{Arc, Weak}, +}; + +use crossbeam_channel::{bounded, Receiver, Sender}; +use tt::Subtree; + +use crate::{ + msg::{ErrorCode, Message, Request, Response, ResponseError}, + rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind}, +}; + +#[derive(Debug, Default)] +pub(crate) struct ProcMacroProcessSrv { + inner: Option>>, +} + +#[derive(Debug)] +pub(crate) struct ProcMacroProcessThread { + // XXX: drop order is significant + sender: Arc>, + handle: jod_thread::JoinHandle<()>, +} + +impl ProcMacroProcessSrv { + pub fn run( + process_path: PathBuf, + args: impl IntoIterator>, + ) -> io::Result<(ProcMacroProcessThread, ProcMacroProcessSrv)> { + let process = Process::run(process_path, args)?; + + let (task_tx, task_rx) = bounded(0); + let handle = jod_thread::spawn(move || { + client_loop(task_rx, process); + }); + + let task_tx = Arc::new(task_tx); + let srv = ProcMacroProcessSrv { inner: Some(Arc::downgrade(&task_tx)) }; + let thread = ProcMacroProcessThread { handle, sender: task_tx }; + + Ok((thread, srv)) + } + + pub fn find_proc_macros( + &self, + dylib_path: &Path, + ) -> Result, tt::ExpansionError> { + let task = ListMacrosTask { lib: dylib_path.to_path_buf() }; + + let result: ListMacrosResult = self.send_task(Request::ListMacro(task))?; + Ok(result.macros) + } + + pub fn custom_derive( + &self, + dylib_path: &Path, + subtree: &Subtree, + derive_name: &str, + ) -> Result { + let task = ExpansionTask { + macro_body: subtree.clone(), + macro_name: derive_name.to_string(), + attributes: None, + lib: dylib_path.to_path_buf(), + }; + + let result: ExpansionResult = self.send_task(Request::ExpansionMacro(task))?; + Ok(result.expansion) + } + + pub fn send_task(&self, req: Request) -> Result + where + R: TryFrom, + { + let sender = match &self.inner { + None => return Err(tt::ExpansionError::Unknown("No sender is found.".to_string())), + Some(it) => it, + }; + + let (result_tx, result_rx) = bounded(0); + let sender = match sender.upgrade() { + None => { + return Err(tt::ExpansionError::Unknown("Proc macro process is closed.".into())) + } + Some(it) => it, + }; + sender.send(Task { req, result_tx }).unwrap(); + let res = result_rx + .recv() + .map_err(|_| tt::ExpansionError::Unknown("Proc macro thread is closed.".into()))?; + + match res { + Some(Response::Error(err)) => { + return Err(tt::ExpansionError::ExpansionError(err.message)); + } + Some(res) => Ok(res.try_into().map_err(|err| { + tt::ExpansionError::Unknown(format!("Fail to get response, reason : {:#?} ", err)) + })?), + None => Err(tt::ExpansionError::Unknown("Empty result".into())), + } + } +} + +fn client_loop(task_rx: Receiver, mut process: Process) { + let (mut stdin, mut stdout) = match process.stdio() { + None => return, + Some(it) => it, + }; + + for task in task_rx { + let Task { req, result_tx } = task; + + match send_request(&mut stdin, &mut stdout, req) { + Ok(res) => result_tx.send(res).unwrap(), + Err(_err) => { + let res = Response::Error(ResponseError { + code: ErrorCode::ServerErrorEnd, + message: "Server closed".into(), + }); + result_tx.send(res.into()).unwrap(); + // Restart the process + if process.restart().is_err() { + break; + } + let stdio = match process.stdio() { + None => break, + Some(it) => it, + }; + stdin = stdio.0; + stdout = stdio.1; + } + } + } +} + +struct Task { + req: Request, + result_tx: Sender>, +} + +struct Process { + path: PathBuf, + args: Vec, + child: Child, +} + +impl Drop for Process { + fn drop(&mut self) { + let _ = self.child.kill(); + } +} + +impl Process { + fn run( + path: PathBuf, + args: impl IntoIterator>, + ) -> io::Result { + let args = args.into_iter().map(|s| s.as_ref().into()).collect(); + let child = mk_child(&path, &args)?; + Ok(Process { path, args, child }) + } + + fn restart(&mut self) -> io::Result<()> { + let _ = self.child.kill(); + self.child = mk_child(&self.path, &self.args)?; + Ok(()) + } + + fn stdio(&mut self) -> Option<(impl Write, impl BufRead)> { + let stdin = self.child.stdin.take()?; + let stdout = self.child.stdout.take()?; + let read = BufReader::new(stdout); + + Some((stdin, read)) + } +} + +fn mk_child(path: &Path, args: impl IntoIterator>) -> io::Result { + Command::new(&path) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() +} + +fn send_request( + mut writer: &mut impl Write, + mut reader: &mut impl BufRead, + req: Request, +) -> io::Result> { + req.write(&mut writer)?; + Ok(Response::read(&mut reader)?) +} diff --git a/crates/proc_macro_api/src/rpc.rs b/crates/proc_macro_api/src/rpc.rs new file mode 100644 index 000000000..47624163e --- /dev/null +++ b/crates/proc_macro_api/src/rpc.rs @@ -0,0 +1,267 @@ +//! Data structure serialization related stuff for RPC +//! +//! Defines all necessary rpc serialization data structures, +//! which includes `tt` related data and some task messages. +//! Although adding `Serialize` and `Deserialize` traits to `tt` directly seems +//! to be much easier, we deliberately duplicate `tt` structs with `#[serde(with = "XXDef")]` +//! for separation of code responsibility. + +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use tt::{ + Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, SmolStr, Spacing, Subtree, TokenId, + TokenTree, +}; + +#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] +pub struct ListMacrosTask { + pub lib: PathBuf, +} + +#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] +pub enum ProcMacroKind { + CustomDerive, + FuncLike, + Attr, +} + +#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)] +pub struct ListMacrosResult { + pub macros: Vec<(String, ProcMacroKind)>, +} + +#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] +pub struct ExpansionTask { + /// Argument of macro call. + /// + /// In custom derive this will be a struct or enum; in attribute-like macro - underlying + /// item; in function-like macro - the macro body. + #[serde(with = "SubtreeDef")] + pub macro_body: Subtree, + + /// Name of macro to expand. + /// + /// In custom derive this is the name of the derived trait (`Serialize`, `Getters`, etc.). + /// In attribute-like and function-like macros - single name of macro itself (`show_streams`). + pub macro_name: String, + + /// Possible attributes for the attribute-like macros. + #[serde(with = "opt_subtree_def")] + pub attributes: Option, + + pub lib: PathBuf, +} + +#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)] +pub struct ExpansionResult { + #[serde(with = "SubtreeDef")] + pub expansion: Subtree, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "DelimiterKind")] +enum DelimiterKindDef { + Parenthesis, + Brace, + Bracket, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "TokenId")] +struct TokenIdDef(u32); + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Delimiter")] +struct DelimiterDef { + #[serde(with = "TokenIdDef")] + pub id: TokenId, + #[serde(with = "DelimiterKindDef")] + pub kind: DelimiterKind, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Subtree")] +struct SubtreeDef { + #[serde(default, with = "opt_delimiter_def")] + pub delimiter: Option, + #[serde(with = "vec_token_tree")] + pub token_trees: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "TokenTree")] +enum TokenTreeDef { + #[serde(with = "LeafDef")] + Leaf(Leaf), + #[serde(with = "SubtreeDef")] + Subtree(Subtree), +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Leaf")] +enum LeafDef { + #[serde(with = "LiteralDef")] + Literal(Literal), + #[serde(with = "PunctDef")] + Punct(Punct), + #[serde(with = "IdentDef")] + Ident(Ident), +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Literal")] +struct LiteralDef { + pub text: SmolStr, + #[serde(with = "TokenIdDef")] + pub id: TokenId, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Punct")] +struct PunctDef { + pub char: char, + #[serde(with = "SpacingDef")] + pub spacing: Spacing, + #[serde(with = "TokenIdDef")] + pub id: TokenId, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Spacing")] +enum SpacingDef { + Alone, + Joint, +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "Ident")] +struct IdentDef { + pub text: SmolStr, + #[serde(with = "TokenIdDef")] + pub id: TokenId, +} + +mod opt_delimiter_def { + use super::{Delimiter, DelimiterDef}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(value: &Option, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "DelimiterDef")] &'a Delimiter); + value.as_ref().map(Helper).serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "DelimiterDef")] Delimiter); + let helper = Option::deserialize(deserializer)?; + Ok(helper.map(|Helper(external)| external)) + } +} + +mod opt_subtree_def { + use super::{Subtree, SubtreeDef}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(value: &Option, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "SubtreeDef")] &'a Subtree); + value.as_ref().map(Helper).serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "SubtreeDef")] Subtree); + let helper = Option::deserialize(deserializer)?; + Ok(helper.map(|Helper(external)| external)) + } +} + +mod vec_token_tree { + use super::{TokenTree, TokenTreeDef}; + use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(value: &Vec, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "TokenTreeDef")] &'a TokenTree); + + let items: Vec<_> = value.iter().map(Helper).collect(); + let mut seq = serializer.serialize_seq(Some(items.len()))?; + for element in items { + seq.serialize_element(&element)?; + } + seq.end() + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "TokenTreeDef")] TokenTree); + + let helper = Vec::deserialize(deserializer)?; + Ok(helper.into_iter().map(|Helper(external)| external).collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture_token_tree() -> Subtree { + let mut subtree = Subtree::default(); + subtree + .token_trees + .push(TokenTree::Leaf(Ident { text: "struct".into(), id: TokenId(0) }.into())); + subtree + .token_trees + .push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into())); + subtree.token_trees.push(TokenTree::Subtree( + Subtree { + delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }), + token_trees: vec![], + } + .into(), + )); + subtree + } + + #[test] + fn test_proc_macro_rpc_works() { + let tt = fixture_token_tree(); + let task = ExpansionTask { + macro_body: tt.clone(), + macro_name: Default::default(), + attributes: None, + lib: Default::default(), + }; + + let json = serde_json::to_string(&task).unwrap(); + let back: ExpansionTask = serde_json::from_str(&json).unwrap(); + + assert_eq!(task.macro_body, back.macro_body); + + let result = ExpansionResult { expansion: tt.clone() }; + let json = serde_json::to_string(&result).unwrap(); + let back: ExpansionResult = serde_json::from_str(&json).unwrap(); + + assert_eq!(result, back); + } +} -- cgit v1.2.3