From ad80a0c551458de7d27a98d182d7f559de04f291 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 31 Jan 2019 18:51:17 +0300 Subject: preserve token spacing --- Cargo.lock | 1 - crates/ra_hir/src/macros.rs | 44 +++++++++++++++++++++++++------------- crates/ra_macros/Cargo.toml | 1 - crates/ra_macros/src/mbe.rs | 2 +- crates/ra_macros/src/mbe_parser.rs | 8 +++---- crates/ra_macros/src/tt.rs | 31 +++++++++++++++++++++------ crates/ra_macros/src/tt_cursor.rs | 2 +- 7 files changed, 59 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9bc80c40..b48721622 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1022,7 +1022,6 @@ dependencies = [ name = "ra_macros" version = "0.1.0" dependencies = [ - "join_to_string 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "smol_str 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/crates/ra_hir/src/macros.rs b/crates/ra_hir/src/macros.rs index 059543bf2..e6ba8c08f 100644 --- a/crates/ra_hir/src/macros.rs +++ b/crates/ra_hir/src/macros.rs @@ -218,14 +218,28 @@ fn convert_tt(tt: &SyntaxNode) -> Option { continue; } if child.kind().is_punct() { - let leaves = child - .leaf_text() - .unwrap() - .chars() - .map(|char| tt::Punct { char }) - .map(tt::Leaf::from) - .map(tt::TokenTree::from); - token_trees.extend(leaves); + let mut prev = None; + for char in child.leaf_text().unwrap().chars() { + if let Some(char) = prev { + token_trees.push( + tt::Leaf::from(tt::Punct { + char, + spacing: tt::Spacing::Joint, + }) + .into(), + ); + } + prev = Some(char) + } + if let Some(char) = prev { + token_trees.push( + tt::Leaf::from(tt::Punct { + char, + spacing: tt::Spacing::Alone, + }) + .into(), + ); + } } else { let child: tt::TokenTree = if child.kind() == TOKEN_TREE { convert_tt(child)?.into() @@ -254,7 +268,7 @@ fn convert_tt(tt: &SyntaxNode) -> Option { #[test] fn test_convert_tt() { - let macro_defenition = r#" + let macro_definition = r#" macro_rules! impl_froms { ($e:ident: $($v:ident),*) => { $( @@ -272,8 +286,8 @@ macro_rules! impl_froms { impl_froms!(TokenTree: Leaf, Subtree); "#; - let source_file = ast::SourceFile::parse(macro_defenition); - let macro_defenition = source_file + let source_file = ast::SourceFile::parse(macro_definition); + let macro_definition = source_file .syntax() .descendants() .find_map(ast::MacroCall::cast) @@ -286,13 +300,13 @@ impl_froms!(TokenTree: Leaf, Subtree); .find_map(ast::MacroCall::cast) .unwrap(); - let defenition_tt = macro_call_to_tt(macro_defenition).unwrap(); + let definition_tt = macro_call_to_tt(macro_definition).unwrap(); let invocation_tt = macro_call_to_tt(macro_invocation).unwrap(); - let mbe = mbe::parse(&defenition_tt).unwrap(); + let mbe = mbe::parse(&definition_tt).unwrap(); let expansion = mbe::exapnd(&mbe, &invocation_tt).unwrap(); assert_eq!( expansion.to_string(), - "{(impl From < Leaf > for TokenTree {fn from (it : Leaf) - > TokenTree {TokenTree : : Leaf (it)}}) \ - (impl From < Subtree > for TokenTree {fn from (it : Subtree) - > TokenTree {TokenTree : : Subtree (it)}})}" + "{(impl From < Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree :: Leaf (it)}}) \ + (impl From < Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree :: Subtree (it)}})}" ) } diff --git a/crates/ra_macros/Cargo.toml b/crates/ra_macros/Cargo.toml index 1c9cc9e92..7d3cb055c 100644 --- a/crates/ra_macros/Cargo.toml +++ b/crates/ra_macros/Cargo.toml @@ -7,4 +7,3 @@ authors = ["Aleksey Kladov "] [dependencies] rustc-hash = "1.0.0" smol_str = "0.1.9" -join_to_string = "0.1.3" diff --git a/crates/ra_macros/src/mbe.rs b/crates/ra_macros/src/mbe.rs index ac76d64e4..d4106a41c 100644 --- a/crates/ra_macros/src/mbe.rs +++ b/crates/ra_macros/src/mbe.rs @@ -45,7 +45,7 @@ pub(crate) struct Subtree { pub(crate) struct Repeat { pub(crate) subtree: Subtree, pub(crate) kind: RepeatKind, - pub(crate) separator: Option, + pub(crate) separator: Option, } #[derive(Debug)] diff --git a/crates/ra_macros/src/mbe_parser.rs b/crates/ra_macros/src/mbe_parser.rs index 279ab2f25..483594590 100644 --- a/crates/ra_macros/src/mbe_parser.rs +++ b/crates/ra_macros/src/mbe_parser.rs @@ -28,16 +28,14 @@ fn parse_subtree(tt: &tt::Subtree) -> Option { while let Some(tt) = p.eat() { let child: mbe::TokenTree = match tt { tt::TokenTree::Leaf(leaf) => match leaf { - tt::Leaf::Punct(tt::Punct { char: '$' }) => { + tt::Leaf::Punct(tt::Punct { char: '$', .. }) => { if p.at_ident().is_some() { mbe::Leaf::from(parse_var(&mut p)?).into() } else { parse_repeat(&mut p)?.into() } } - tt::Leaf::Punct(tt::Punct { char }) => { - mbe::Leaf::from(mbe::Punct { char: *char }).into() - } + tt::Leaf::Punct(punct) => mbe::Leaf::from(*punct).into(), tt::Leaf::Ident(tt::Ident { text }) => { mbe::Leaf::from(mbe::Ident { text: text.clone() }).into() } @@ -78,7 +76,7 @@ fn parse_repeat(p: &mut TtCursor) -> Option { let sep = p.eat_punct()?; let (separator, rep) = match sep.char { '*' | '+' | '?' => (None, sep.char), - char => (Some(mbe::Punct { char }), p.eat_punct()?.char), + char => (Some(char), p.eat_punct()?.char), }; let kind = match rep { diff --git a/crates/ra_macros/src/tt.rs b/crates/ra_macros/src/tt.rs index 56e930f00..2855bae51 100644 --- a/crates/ra_macros/src/tt.rs +++ b/crates/ra_macros/src/tt.rs @@ -1,7 +1,6 @@ use std::fmt; use smol_str::SmolStr; -use join_to_string::join; #[derive(Debug, Clone)] pub enum TokenTree { @@ -37,9 +36,16 @@ pub struct Literal { pub text: SmolStr, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Punct { pub char: char, + pub spacing: Spacing, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Spacing { + Alone, + Joint, } #[derive(Debug, Clone)] @@ -64,10 +70,23 @@ impl fmt::Display for Subtree { Delimiter::Bracket => ("[", "]"), Delimiter::None => ("", ""), }; - join(self.token_trees.iter()) - .separator(" ") - .surround_with(l, r) - .to_fmt(f) + f.write_str(l)?; + let mut needs_space = false; + for tt in self.token_trees.iter() { + if needs_space { + f.write_str(" ")?; + } + needs_space = true; + match tt { + TokenTree::Leaf(Leaf::Punct(p)) => { + needs_space = p.spacing == Spacing::Alone; + fmt::Display::fmt(p, f)? + } + tt => fmt::Display::fmt(tt, f)?, + } + } + f.write_str(r)?; + Ok(()) } } diff --git a/crates/ra_macros/src/tt_cursor.rs b/crates/ra_macros/src/tt_cursor.rs index 6dc9f400d..046770a0f 100644 --- a/crates/ra_macros/src/tt_cursor.rs +++ b/crates/ra_macros/src/tt_cursor.rs @@ -28,7 +28,7 @@ impl<'a> TtCursor<'a> { pub(crate) fn at_char(&self, char: char) -> bool { match self.at_punct() { - Some(tt::Punct { char: c }) if *c == char => true, + Some(tt::Punct { char: c, .. }) if *c == char => true, _ => false, } } -- cgit v1.2.3