preserve token spacing

7 files changed, 59 insertions, 30 deletions

diff --git a/Cargo.lock b/Cargo.lock
index a9bc80c40..b48721622 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1022,7 +1022,6 @@ dependencies = [
 name = "ra_macros"
 version = "0.1.0"
 dependencies = [
- "join_to_string 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "smol_str 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
diff --git a/crates/ra_hir/src/macros.rs b/crates/ra_hir/src/macros.rs
index 059543bf2..e6ba8c08f 100644
--- a/crates/ra_hir/src/macros.rs
+++ b/crates/ra_hir/src/macros.rs
@@ -218,14 +218,28 @@ fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
             continue;
         }
         if child.kind().is_punct() {
-            let leaves = child
-                .leaf_text()
-                .unwrap()
-                .chars()
-                .map(|char| tt::Punct { char })
-                .map(tt::Leaf::from)
-                .map(tt::TokenTree::from);
-            token_trees.extend(leaves);
+            let mut prev = None;
+            for char in child.leaf_text().unwrap().chars() {
+                if let Some(char) = prev {
+                    token_trees.push(
+                        tt::Leaf::from(tt::Punct {
+                            char,
+                            spacing: tt::Spacing::Joint,
+                        })
+                        .into(),
+                    );
+                }
+                prev = Some(char)
+            }
+            if let Some(char) = prev {
+                token_trees.push(
+                    tt::Leaf::from(tt::Punct {
+                        char,
+                        spacing: tt::Spacing::Alone,
+                    })
+                    .into(),
+                );
+            }
         } else {
             let child: tt::TokenTree = if child.kind() == TOKEN_TREE {
                 convert_tt(child)?.into()
@@ -254,7 +268,7 @@ fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
 
 #[test]
 fn test_convert_tt() {
-    let macro_defenition = r#"
+    let macro_definition = r#"
 macro_rules! impl_froms {
     ($e:ident: $($v:ident),*) => {
         $(
@@ -272,8 +286,8 @@ macro_rules! impl_froms {
 impl_froms!(TokenTree: Leaf, Subtree);
 "#;
 
-    let source_file = ast::SourceFile::parse(macro_defenition);
-    let macro_defenition = source_file
+    let source_file = ast::SourceFile::parse(macro_definition);
+    let macro_definition = source_file
         .syntax()
         .descendants()
         .find_map(ast::MacroCall::cast)
@@ -286,13 +300,13 @@ impl_froms!(TokenTree: Leaf, Subtree);
         .find_map(ast::MacroCall::cast)
         .unwrap();
 
-    let defenition_tt = macro_call_to_tt(macro_defenition).unwrap();
+    let definition_tt = macro_call_to_tt(macro_definition).unwrap();
     let invocation_tt = macro_call_to_tt(macro_invocation).unwrap();
-    let mbe = mbe::parse(&defenition_tt).unwrap();
+    let mbe = mbe::parse(&definition_tt).unwrap();
     let expansion = mbe::exapnd(&mbe, &invocation_tt).unwrap();
     assert_eq!(
         expansion.to_string(),
-        "{(impl From < Leaf > for TokenTree {fn from (it : Leaf) - > TokenTree {TokenTree : : Leaf (it)}}) \
-          (impl From < Subtree > for TokenTree {fn from (it : Subtree) - > TokenTree {TokenTree : : Subtree (it)}})}"
+        "{(impl From < Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree :: Leaf (it)}}) \
+          (impl From < Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree :: Subtree (it)}})}"
     )
 }
diff --git a/crates/ra_macros/Cargo.toml b/crates/ra_macros/Cargo.toml
index 1c9cc9e92..7d3cb055c 100644
--- a/crates/ra_macros/Cargo.toml
+++ b/crates/ra_macros/Cargo.toml
@@ -7,4 +7,3 @@ authors = ["Aleksey Kladov <[email protected]>"]
 [dependencies]
 rustc-hash = "1.0.0"
 smol_str = "0.1.9"
-join_to_string = "0.1.3"
diff --git a/crates/ra_macros/src/mbe.rs b/crates/ra_macros/src/mbe.rs
index ac76d64e4..d4106a41c 100644
--- a/crates/ra_macros/src/mbe.rs
+++ b/crates/ra_macros/src/mbe.rs
@@ -45,7 +45,7 @@ pub(crate) struct Subtree {
 pub(crate) struct Repeat {
     pub(crate) subtree: Subtree,
     pub(crate) kind: RepeatKind,
-    pub(crate) separator: Option<Punct>,
+    pub(crate) separator: Option<char>,
 }
 
 #[derive(Debug)]
diff --git a/crates/ra_macros/src/mbe_parser.rs b/crates/ra_macros/src/mbe_parser.rs
index 279ab2f25..483594590 100644
--- a/crates/ra_macros/src/mbe_parser.rs
+++ b/crates/ra_macros/src/mbe_parser.rs
@@ -28,16 +28,14 @@ fn parse_subtree(tt: &tt::Subtree) -> Option<mbe::Subtree> {
     while let Some(tt) = p.eat() {
         let child: mbe::TokenTree = match tt {
             tt::TokenTree::Leaf(leaf) => match leaf {
-                tt::Leaf::Punct(tt::Punct { char: '$' }) => {
+                tt::Leaf::Punct(tt::Punct { char: '$', .. }) => {
                     if p.at_ident().is_some() {
                         mbe::Leaf::from(parse_var(&mut p)?).into()
                     } else {
                         parse_repeat(&mut p)?.into()
                     }
                 }
-                tt::Leaf::Punct(tt::Punct { char }) => {
-                    mbe::Leaf::from(mbe::Punct { char: *char }).into()
-                }
+                tt::Leaf::Punct(punct) => mbe::Leaf::from(*punct).into(),
                 tt::Leaf::Ident(tt::Ident { text }) => {
                     mbe::Leaf::from(mbe::Ident { text: text.clone() }).into()
                 }
@@ -78,7 +76,7 @@ fn parse_repeat(p: &mut TtCursor) -> Option<mbe::Repeat> {
     let sep = p.eat_punct()?;
     let (separator, rep) = match sep.char {
         '*' | '+' | '?' => (None, sep.char),
-        char => (Some(mbe::Punct { char }), p.eat_punct()?.char),
+        char => (Some(char), p.eat_punct()?.char),
     };
 
     let kind = match rep {
diff --git a/crates/ra_macros/src/tt.rs b/crates/ra_macros/src/tt.rs
index 56e930f00..2855bae51 100644
--- a/crates/ra_macros/src/tt.rs
+++ b/crates/ra_macros/src/tt.rs
@@ -1,7 +1,6 @@
 use std::fmt;
 
 use smol_str::SmolStr;
-use join_to_string::join;
 
 #[derive(Debug, Clone)]
 pub enum TokenTree {
@@ -37,9 +36,16 @@ pub struct Literal {
     pub text: SmolStr,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Punct {
     pub char: char,
+    pub spacing: Spacing,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Spacing {
+    Alone,
+    Joint,
 }
 
 #[derive(Debug, Clone)]
@@ -64,10 +70,23 @@ impl fmt::Display for Subtree {
             Delimiter::Bracket => ("[", "]"),
             Delimiter::None => ("", ""),
         };
-        join(self.token_trees.iter())
-            .separator(" ")
-            .surround_with(l, r)
-            .to_fmt(f)
+        f.write_str(l)?;
+        let mut needs_space = false;
+        for tt in self.token_trees.iter() {
+            if needs_space {
+                f.write_str(" ")?;
+            }
+            needs_space = true;
+            match tt {
+                TokenTree::Leaf(Leaf::Punct(p)) => {
+                    needs_space = p.spacing == Spacing::Alone;
+                    fmt::Display::fmt(p, f)?
+                }
+                tt => fmt::Display::fmt(tt, f)?,
+            }
+        }
+        f.write_str(r)?;
+        Ok(())
     }
 }
 
diff --git a/crates/ra_macros/src/tt_cursor.rs b/crates/ra_macros/src/tt_cursor.rs
index 6dc9f400d..046770a0f 100644
--- a/crates/ra_macros/src/tt_cursor.rs
+++ b/crates/ra_macros/src/tt_cursor.rs
@@ -28,7 +28,7 @@ impl<'a> TtCursor<'a> {
 
     pub(crate) fn at_char(&self, char: char) -> bool {
         match self.at_punct() {
-            Some(tt::Punct { char: c }) if *c == char => true,
+            Some(tt::Punct { char: c, .. }) if *c == char => true,
             _ => false,
         }
     }