Merge remote-tracking branch 'origin/master'

5 files changed, 686 insertions, 0 deletions

diff --git a/crates/proc_macro_api/Cargo.toml b/crates/proc_macro_api/Cargo.toml
new file mode 100644
index 000000000..a3a4c1103
--- /dev/null
+++ b/crates/proc_macro_api/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+name = "proc_macro_api"
+version = "0.0.0"
+license = "MIT OR Apache-2.0"
+authors = ["rust-analyzer developers"]
+edition = "2018"
+
+[lib]
+doctest = false
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+log = "0.4.8"
+crossbeam-channel = "0.4.0"
+jod-thread = "0.1.1"
+
+tt = { path = "../tt" }
diff --git a/crates/proc_macro_api/src/lib.rs b/crates/proc_macro_api/src/lib.rs
new file mode 100644
index 000000000..15db57eb2
--- /dev/null
+++ b/crates/proc_macro_api/src/lib.rs
@@ -0,0 +1,111 @@
+//! Client-side Proc-Macro crate
+//!
+//! We separate proc-macro expanding logic to an extern program to allow
+//! different implementations (e.g. wasm or dylib loading). And this crate
+//! is used to provide basic infrastructure for communication between two
+//! processes: Client (RA itself), Server (the external program)
+
+mod rpc;
+mod process;
+pub mod msg;
+
+use std::{
+    ffi::OsStr,
+    io,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use tt::{SmolStr, Subtree};
+
+use crate::process::{ProcMacroProcessSrv, ProcMacroProcessThread};
+
+pub use rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind};
+
+#[derive(Debug, Clone)]
+pub struct ProcMacroProcessExpander {
+    process: Arc<ProcMacroProcessSrv>,
+    dylib_path: PathBuf,
+    name: SmolStr,
+}
+
+impl Eq for ProcMacroProcessExpander {}
+impl PartialEq for ProcMacroProcessExpander {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+            && self.dylib_path == other.dylib_path
+            && Arc::ptr_eq(&self.process, &other.process)
+    }
+}
+
+impl tt::TokenExpander for ProcMacroProcessExpander {
+    fn expand(
+        &self,
+        subtree: &Subtree,
+        _attr: Option<&Subtree>,
+    ) -> Result<Subtree, tt::ExpansionError> {
+        self.process.custom_derive(&self.dylib_path, subtree, &self.name)
+    }
+}
+
+#[derive(Debug)]
+enum ProcMacroClientKind {
+    Process { process: Arc<ProcMacroProcessSrv>, thread: ProcMacroProcessThread },
+    Dummy,
+}
+
+#[derive(Debug)]
+pub struct ProcMacroClient {
+    kind: ProcMacroClientKind,
+}
+
+impl ProcMacroClient {
+    pub fn extern_process(
+        process_path: PathBuf,
+        args: impl IntoIterator<Item = impl AsRef<OsStr>>,
+    ) -> io::Result<ProcMacroClient> {
+        let (thread, process) = ProcMacroProcessSrv::run(process_path, args)?;
+        Ok(ProcMacroClient {
+            kind: ProcMacroClientKind::Process { process: Arc::new(process), thread },
+        })
+    }
+
+    pub fn dummy() -> ProcMacroClient {
+        ProcMacroClient { kind: ProcMacroClientKind::Dummy }
+    }
+
+    pub fn by_dylib_path(&self, dylib_path: &Path) -> Vec<(SmolStr, Arc<dyn tt::TokenExpander>)> {
+        match &self.kind {
+            ProcMacroClientKind::Dummy => vec![],
+            ProcMacroClientKind::Process { process, .. } => {
+                let macros = match process.find_proc_macros(dylib_path) {
+                    Err(err) => {
+                        eprintln!("Failed to find proc macros. Error: {:#?}", err);
+                        return vec![];
+                    }
+                    Ok(macros) => macros,
+                };
+
+                macros
+                    .into_iter()
+                    .filter_map(|(name, kind)| {
+                        // FIXME: Support custom derive only for now.
+                        match kind {
+                            ProcMacroKind::CustomDerive => {
+                                let name = SmolStr::new(&name);
+                                let expander: Arc<dyn tt::TokenExpander> =
+                                    Arc::new(ProcMacroProcessExpander {
+                                        process: process.clone(),
+                                        name: name.clone(),
+                                        dylib_path: dylib_path.into(),
+                                    });
+                                Some((name, expander))
+                            }
+                            _ => None,
+                        }
+                    })
+                    .collect()
+            }
+        }
+    }
+}
diff --git a/crates/proc_macro_api/src/msg.rs b/crates/proc_macro_api/src/msg.rs
new file mode 100644
index 000000000..f84ebdbc5
--- /dev/null
+++ b/crates/proc_macro_api/src/msg.rs
@@ -0,0 +1,89 @@
+//! Defines messages for cross-process message passing based on `ndjson` wire protocol
+
+use std::{
+    convert::TryFrom,
+    io::{self, BufRead, Write},
+};
+
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+
+use crate::{
+    rpc::{ListMacrosResult, ListMacrosTask},
+    ExpansionResult, ExpansionTask,
+};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Request {
+    ListMacro(ListMacrosTask),
+    ExpansionMacro(ExpansionTask),
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Response {
+    Error(ResponseError),
+    ListMacro(ListMacrosResult),
+    ExpansionMacro(ExpansionResult),
+}
+
+macro_rules! impl_try_from_response {
+    ($ty:ty, $tag:ident) => {
+        impl TryFrom<Response> for $ty {
+            type Error = &'static str;
+            fn try_from(value: Response) -> Result<Self, Self::Error> {
+                match value {
+                    Response::$tag(res) => Ok(res),
+                    _ => Err(concat!("Failed to convert response to ", stringify!($tag))),
+                }
+            }
+        }
+    };
+}
+
+impl_try_from_response!(ListMacrosResult, ListMacro);
+impl_try_from_response!(ExpansionResult, ExpansionMacro);
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct ResponseError {
+    pub code: ErrorCode,
+    pub message: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum ErrorCode {
+    ServerErrorEnd,
+    ExpansionError,
+}
+
+pub trait Message: Serialize + DeserializeOwned {
+    fn read(inp: &mut impl BufRead) -> io::Result<Option<Self>> {
+        Ok(match read_json(inp)? {
+            None => None,
+            Some(text) => Some(serde_json::from_str(&text)?),
+        })
+    }
+    fn write(self, out: &mut impl Write) -> io::Result<()> {
+        let text = serde_json::to_string(&self)?;
+        write_json(out, &text)
+    }
+}
+
+impl Message for Request {}
+impl Message for Response {}
+
+fn read_json(inp: &mut impl BufRead) -> io::Result<Option<String>> {
+    let mut buf = String::new();
+    inp.read_line(&mut buf)?;
+    buf.pop(); // Remove traling '\n'
+    Ok(match buf.len() {
+        0 => None,
+        _ => Some(buf),
+    })
+}
+
+fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> {
+    log::debug!("> {}", msg);
+    out.write_all(msg.as_bytes())?;
+    out.write_all(b"\n")?;
+    out.flush()?;
+    Ok(())
+}
diff --git a/crates/proc_macro_api/src/process.rs b/crates/proc_macro_api/src/process.rs
new file mode 100644
index 000000000..51ffcaa78
--- /dev/null
+++ b/crates/proc_macro_api/src/process.rs
@@ -0,0 +1,201 @@
+//! Handle process life-time and message passing for proc-macro client
+
+use std::{
+    convert::{TryFrom, TryInto},
+    ffi::{OsStr, OsString},
+    io::{self, BufRead, BufReader, Write},
+    path::{Path, PathBuf},
+    process::{Child, Command, Stdio},
+    sync::{Arc, Weak},
+};
+
+use crossbeam_channel::{bounded, Receiver, Sender};
+use tt::Subtree;
+
+use crate::{
+    msg::{ErrorCode, Message, Request, Response, ResponseError},
+    rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind},
+};
+
+#[derive(Debug, Default)]
+pub(crate) struct ProcMacroProcessSrv {
+    inner: Option<Weak<Sender<Task>>>,
+}
+
+#[derive(Debug)]
+pub(crate) struct ProcMacroProcessThread {
+    // XXX: drop order is significant
+    sender: Arc<Sender<Task>>,
+    handle: jod_thread::JoinHandle<()>,
+}
+
+impl ProcMacroProcessSrv {
+    pub fn run(
+        process_path: PathBuf,
+        args: impl IntoIterator<Item = impl AsRef<OsStr>>,
+    ) -> io::Result<(ProcMacroProcessThread, ProcMacroProcessSrv)> {
+        let process = Process::run(process_path, args)?;
+
+        let (task_tx, task_rx) = bounded(0);
+        let handle = jod_thread::spawn(move || {
+            client_loop(task_rx, process);
+        });
+
+        let task_tx = Arc::new(task_tx);
+        let srv = ProcMacroProcessSrv { inner: Some(Arc::downgrade(&task_tx)) };
+        let thread = ProcMacroProcessThread { handle, sender: task_tx };
+
+        Ok((thread, srv))
+    }
+
+    pub fn find_proc_macros(
+        &self,
+        dylib_path: &Path,
+    ) -> Result<Vec<(String, ProcMacroKind)>, tt::ExpansionError> {
+        let task = ListMacrosTask { lib: dylib_path.to_path_buf() };
+
+        let result: ListMacrosResult = self.send_task(Request::ListMacro(task))?;
+        Ok(result.macros)
+    }
+
+    pub fn custom_derive(
+        &self,
+        dylib_path: &Path,
+        subtree: &Subtree,
+        derive_name: &str,
+    ) -> Result<Subtree, tt::ExpansionError> {
+        let task = ExpansionTask {
+            macro_body: subtree.clone(),
+            macro_name: derive_name.to_string(),
+            attributes: None,
+            lib: dylib_path.to_path_buf(),
+        };
+
+        let result: ExpansionResult = self.send_task(Request::ExpansionMacro(task))?;
+        Ok(result.expansion)
+    }
+
+    pub fn send_task<R>(&self, req: Request) -> Result<R, tt::ExpansionError>
+    where
+        R: TryFrom<Response, Error = &'static str>,
+    {
+        let sender = match &self.inner {
+            None => return Err(tt::ExpansionError::Unknown("No sender is found.".to_string())),
+            Some(it) => it,
+        };
+
+        let (result_tx, result_rx) = bounded(0);
+        let sender = match sender.upgrade() {
+            None => {
+                return Err(tt::ExpansionError::Unknown("Proc macro process is closed.".into()))
+            }
+            Some(it) => it,
+        };
+        sender.send(Task { req, result_tx }).unwrap();
+        let res = result_rx
+            .recv()
+            .map_err(|_| tt::ExpansionError::Unknown("Proc macro thread is closed.".into()))?;
+
+        match res {
+            Some(Response::Error(err)) => {
+                return Err(tt::ExpansionError::ExpansionError(err.message));
+            }
+            Some(res) => Ok(res.try_into().map_err(|err| {
+                tt::ExpansionError::Unknown(format!("Fail to get response, reason : {:#?} ", err))
+            })?),
+            None => Err(tt::ExpansionError::Unknown("Empty result".into())),
+        }
+    }
+}
+
+fn client_loop(task_rx: Receiver<Task>, mut process: Process) {
+    let (mut stdin, mut stdout) = match process.stdio() {
+        None => return,
+        Some(it) => it,
+    };
+
+    for task in task_rx {
+        let Task { req, result_tx } = task;
+
+        match send_request(&mut stdin, &mut stdout, req) {
+            Ok(res) => result_tx.send(res).unwrap(),
+            Err(_err) => {
+                let res = Response::Error(ResponseError {
+                    code: ErrorCode::ServerErrorEnd,
+                    message: "Server closed".into(),
+                });
+                result_tx.send(res.into()).unwrap();
+                // Restart the process
+                if process.restart().is_err() {
+                    break;
+                }
+                let stdio = match process.stdio() {
+                    None => break,
+                    Some(it) => it,
+                };
+                stdin = stdio.0;
+                stdout = stdio.1;
+            }
+        }
+    }
+}
+
+struct Task {
+    req: Request,
+    result_tx: Sender<Option<Response>>,
+}
+
+struct Process {
+    path: PathBuf,
+    args: Vec<OsString>,
+    child: Child,
+}
+
+impl Drop for Process {
+    fn drop(&mut self) {
+        let _ = self.child.kill();
+    }
+}
+
+impl Process {
+    fn run(
+        path: PathBuf,
+        args: impl IntoIterator<Item = impl AsRef<OsStr>>,
+    ) -> io::Result<Process> {
+        let args = args.into_iter().map(|s| s.as_ref().into()).collect();
+        let child = mk_child(&path, &args)?;
+        Ok(Process { path, args, child })
+    }
+
+    fn restart(&mut self) -> io::Result<()> {
+        let _ = self.child.kill();
+        self.child = mk_child(&self.path, &self.args)?;
+        Ok(())
+    }
+
+    fn stdio(&mut self) -> Option<(impl Write, impl BufRead)> {
+        let stdin = self.child.stdin.take()?;
+        let stdout = self.child.stdout.take()?;
+        let read = BufReader::new(stdout);
+
+        Some((stdin, read))
+    }
+}
+
+fn mk_child(path: &Path, args: impl IntoIterator<Item = impl AsRef<OsStr>>) -> io::Result<Child> {
+    Command::new(&path)
+        .args(args)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::inherit())
+        .spawn()
+}
+
+fn send_request(
+    mut writer: &mut impl Write,
+    mut reader: &mut impl BufRead,
+    req: Request,
+) -> io::Result<Option<Response>> {
+    req.write(&mut writer)?;
+    Ok(Response::read(&mut reader)?)
+}
diff --git a/crates/proc_macro_api/src/rpc.rs b/crates/proc_macro_api/src/rpc.rs
new file mode 100644
index 000000000..47624163e
--- /dev/null
+++ b/crates/proc_macro_api/src/rpc.rs
@@ -0,0 +1,267 @@
+//! Data structure serialization related stuff for RPC
+//!
+//! Defines all necessary rpc serialization data structures,
+//! which includes `tt` related data and some task messages.
+//! Although adding `Serialize` and `Deserialize` traits to `tt` directly seems
+//! to be much easier, we deliberately duplicate `tt` structs with `#[serde(with = "XXDef")]`
+//! for separation of code responsibility.
+
+use std::path::PathBuf;
+
+use serde::{Deserialize, Serialize};
+use tt::{
+    Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, SmolStr, Spacing, Subtree, TokenId,
+    TokenTree,
+};
+
+#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)]
+pub struct ListMacrosTask {
+    pub lib: PathBuf,
+}
+
+#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)]
+pub enum ProcMacroKind {
+    CustomDerive,
+    FuncLike,
+    Attr,
+}
+
+#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)]
+pub struct ListMacrosResult {
+    pub macros: Vec<(String, ProcMacroKind)>,
+}
+
+#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)]
+pub struct ExpansionTask {
+    /// Argument of macro call.
+    ///
+    /// In custom derive this will be a struct or enum; in attribute-like macro - underlying
+    /// item; in function-like macro - the macro body.
+    #[serde(with = "SubtreeDef")]
+    pub macro_body: Subtree,
+
+    /// Name of macro to expand.
+    ///
+    /// In custom derive this is the name of the derived trait (`Serialize`, `Getters`, etc.).
+    /// In attribute-like and function-like macros - single name of macro itself (`show_streams`).
+    pub macro_name: String,
+
+    /// Possible attributes for the attribute-like macros.
+    #[serde(with = "opt_subtree_def")]
+    pub attributes: Option<Subtree>,
+
+    pub lib: PathBuf,
+}
+
+#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)]
+pub struct ExpansionResult {
+    #[serde(with = "SubtreeDef")]
+    pub expansion: Subtree,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "DelimiterKind")]
+enum DelimiterKindDef {
+    Parenthesis,
+    Brace,
+    Bracket,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "TokenId")]
+struct TokenIdDef(u32);
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "Delimiter")]
+struct DelimiterDef {
+    #[serde(with = "TokenIdDef")]
+    pub id: TokenId,
+    #[serde(with = "DelimiterKindDef")]
+    pub kind: DelimiterKind,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "Subtree")]
+struct SubtreeDef {
+    #[serde(default, with = "opt_delimiter_def")]
+    pub delimiter: Option<Delimiter>,
+    #[serde(with = "vec_token_tree")]
+    pub token_trees: Vec<TokenTree>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "TokenTree")]
+enum TokenTreeDef {
+    #[serde(with = "LeafDef")]
+    Leaf(Leaf),
+    #[serde(with = "SubtreeDef")]
+    Subtree(Subtree),
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "Leaf")]
+enum LeafDef {
+    #[serde(with = "LiteralDef")]
+    Literal(Literal),
+    #[serde(with = "PunctDef")]
+    Punct(Punct),
+    #[serde(with = "IdentDef")]
+    Ident(Ident),
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "Literal")]
+struct LiteralDef {
+    pub text: SmolStr,
+    #[serde(with = "TokenIdDef")]
+    pub id: TokenId,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "Punct")]
+struct PunctDef {
+    pub char: char,
+    #[serde(with = "SpacingDef")]
+    pub spacing: Spacing,
+    #[serde(with = "TokenIdDef")]
+    pub id: TokenId,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "Spacing")]
+enum SpacingDef {
+    Alone,
+    Joint,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "Ident")]
+struct IdentDef {
+    pub text: SmolStr,
+    #[serde(with = "TokenIdDef")]
+    pub id: TokenId,
+}
+
+mod opt_delimiter_def {
+    use super::{Delimiter, DelimiterDef};
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+    pub fn serialize<S>(value: &Option<Delimiter>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        #[derive(Serialize)]
+        struct Helper<'a>(#[serde(with = "DelimiterDef")] &'a Delimiter);
+        value.as_ref().map(Helper).serialize(serializer)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Delimiter>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        struct Helper(#[serde(with = "DelimiterDef")] Delimiter);
+        let helper = Option::deserialize(deserializer)?;
+        Ok(helper.map(|Helper(external)| external))
+    }
+}
+
+mod opt_subtree_def {
+    use super::{Subtree, SubtreeDef};
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+    pub fn serialize<S>(value: &Option<Subtree>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        #[derive(Serialize)]
+        struct Helper<'a>(#[serde(with = "SubtreeDef")] &'a Subtree);
+        value.as_ref().map(Helper).serialize(serializer)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Subtree>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        struct Helper(#[serde(with = "SubtreeDef")] Subtree);
+        let helper = Option::deserialize(deserializer)?;
+        Ok(helper.map(|Helper(external)| external))
+    }
+}
+
+mod vec_token_tree {
+    use super::{TokenTree, TokenTreeDef};
+    use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer};
+
+    pub fn serialize<S>(value: &Vec<TokenTree>, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        #[derive(Serialize)]
+        struct Helper<'a>(#[serde(with = "TokenTreeDef")] &'a TokenTree);
+
+        let items: Vec<_> = value.iter().map(Helper).collect();
+        let mut seq = serializer.serialize_seq(Some(items.len()))?;
+        for element in items {
+            seq.serialize_element(&element)?;
+        }
+        seq.end()
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<TokenTree>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        struct Helper(#[serde(with = "TokenTreeDef")] TokenTree);
+
+        let helper = Vec::deserialize(deserializer)?;
+        Ok(helper.into_iter().map(|Helper(external)| external).collect())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn fixture_token_tree() -> Subtree {
+        let mut subtree = Subtree::default();
+        subtree
+            .token_trees
+            .push(TokenTree::Leaf(Ident { text: "struct".into(), id: TokenId(0) }.into()));
+        subtree
+            .token_trees
+            .push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into()));
+        subtree.token_trees.push(TokenTree::Subtree(
+            Subtree {
+                delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }),
+                token_trees: vec![],
+            }
+            .into(),
+        ));
+        subtree
+    }
+
+    #[test]
+    fn test_proc_macro_rpc_works() {
+        let tt = fixture_token_tree();
+        let task = ExpansionTask {
+            macro_body: tt.clone(),
+            macro_name: Default::default(),
+            attributes: None,
+            lib: Default::default(),
+        };
+
+        let json = serde_json::to_string(&task).unwrap();
+        let back: ExpansionTask = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(task.macro_body, back.macro_body);
+
+        let result = ExpansionResult { expansion: tt.clone() };
+        let json = serde_json::to_string(&result).unwrap();
+        let back: ExpansionResult = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(result, back);
+    }
+}