From aceb9d7fb0809ccf364514d9177342edea144c59 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Thu, 12 Dec 2019 21:47:54 +0800 Subject: Add token ids for all tt::Leaf --- crates/ra_hir_expand/src/builtin_derive.rs | 28 ++++++++++++++--- crates/ra_hir_expand/src/quote.rs | 13 +++++--- crates/ra_mbe/src/mbe_expander/transcriber.rs | 7 ++++- crates/ra_mbe/src/syntax_bridge.rs | 45 ++++++++++++++++----------- crates/ra_mbe/src/tests.rs | 10 +++--- crates/ra_tt/src/lib.rs | 2 ++ 6 files changed, 72 insertions(+), 33 deletions(-) (limited to 'crates') diff --git a/crates/ra_hir_expand/src/builtin_derive.rs b/crates/ra_hir_expand/src/builtin_derive.rs index b26441253..62c60e336 100644 --- a/crates/ra_hir_expand/src/builtin_derive.rs +++ b/crates/ra_hir_expand/src/builtin_derive.rs @@ -97,11 +97,24 @@ fn parse_adt(tt: &tt::Subtree) -> Result { fn make_type_args(n: usize, bound: Vec) -> Vec { let mut result = Vec::::new(); - result.push(tt::Leaf::Punct(tt::Punct { char: '<', spacing: tt::Spacing::Alone }).into()); + result.push( + tt::Leaf::Punct(tt::Punct { + char: '<', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), + ); for i in 0..n { if i > 0 { - result - .push(tt::Leaf::Punct(tt::Punct { char: ',', spacing: tt::Spacing::Alone }).into()); + result.push( + tt::Leaf::Punct(tt::Punct { + char: ',', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), + ); } result.push( tt::Leaf::Ident(tt::Ident { @@ -112,7 +125,14 @@ fn make_type_args(n: usize, bound: Vec) -> Vec { ); result.extend(bound.iter().cloned()); } - result.push(tt::Leaf::Punct(tt::Punct { char: '>', spacing: tt::Spacing::Alone }).into()); + result.push( + tt::Leaf::Punct(tt::Punct { + char: '>', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), + ); result } diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs index aa8a5f23f..bce38cc67 100644 --- a/crates/ra_hir_expand/src/quote.rs +++ b/crates/ra_hir_expand/src/quote.rs @@ -29,6 +29,7 @@ macro_rules! __quote { tt::Leaf::Punct(tt::Punct { char: $first, spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), }).into() ] } @@ -40,10 +41,12 @@ macro_rules! __quote { tt::Leaf::Punct(tt::Punct { char: $first, spacing: tt::Spacing::Joint, + id: tt::TokenId::unspecified(), }).into(), tt::Leaf::Punct(tt::Punct { char: $sec, spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), }).into() ] } @@ -179,15 +182,15 @@ macro_rules! impl_to_to_tokentrees { } impl_to_to_tokentrees! { - u32 => self { tt::Literal{text: self.to_string().into()} }; - usize => self { tt::Literal{text: self.to_string().into()}}; - i32 => self { tt::Literal{text: self.to_string().into()}}; + u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; + usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()}}; + i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()}}; tt::Leaf => self { self }; tt::Literal => self { self }; tt::Ident => self { self }; tt::Punct => self { self }; - &str => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into()}}; - String => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into()}} + &str => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into(), id: tt::TokenId::unspecified()}}; + String => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into(), id: tt::TokenId::unspecified()}} } #[cfg(test)] diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index f7636db11..eda66cd50 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs @@ -108,7 +108,12 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result let tt = tt::Subtree { delimiter: None, token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(), + tt::Leaf::from(tt::Punct { + char: '$', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) .into(), ], diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index b8e2cfc1d..8f65ff125 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -136,11 +136,15 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option tt::TokenTree { - tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone })) + tt::TokenTree::from(tt::Leaf::from(tt::Punct { + char: c, + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + })) } fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree { - let lit = tt::Literal { text: doc_comment_text(comment) }; + let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() }; tt::TokenTree::from(tt::Leaf::from(lit)) } @@ -223,24 +227,29 @@ impl Convertor { .take(token.text().len() - 1) .chain(std::iter::once(last_spacing)); for (char, spacing) in token.text().chars().zip(spacing_iter) { - token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into()); + let id = self.alloc(token.text_range()); + token_trees + .push(tt::Leaf::from(tt::Punct { char, spacing, id }).into()); } } else { - let child: tt::TokenTree = - if token.kind() == T![true] || token.kind() == T![false] { - tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() - } else if token.kind().is_keyword() - || token.kind() == IDENT - || token.kind() == LIFETIME - { - let id = self.alloc(token.text_range()); - let text = token.text().clone(); - tt::Leaf::from(tt::Ident { text, id }).into() - } else if token.kind().is_literal() { - tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() - } else { - return None; - }; + let child: tt::TokenTree = if token.kind() == T![true] + || token.kind() == T![false] + { + let id = self.alloc(token.text_range()); + tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() + } else if token.kind().is_keyword() + || token.kind() == IDENT + || token.kind() == LIFETIME + { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Leaf::from(tt::Ident { text, id }).into() + } else if token.kind().is_literal() { + let id = self.alloc(token.text_range()); + tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() + } else { + return None; + }; token_trees.push(child); } } diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 148cc2625..70e65bc74 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -78,12 +78,12 @@ macro_rules! foobar { assert_eq!(expansion.token_trees.len(), 3); // ($e:ident) => { foo bar $e } - // 0 1 2 3 4 - assert_eq!(get_id(&expansion.token_trees[0]), Some(2)); - assert_eq!(get_id(&expansion.token_trees[1]), Some(3)); + // 0123 45 6 7 89 + assert_eq!(get_id(&expansion.token_trees[0]), Some(6)); + assert_eq!(get_id(&expansion.token_trees[1]), Some(7)); - // So baz should be 5 - assert_eq!(get_id(&expansion.token_trees[2]), Some(5)); + // So baz should be 10 + assert_eq!(get_id(&expansion.token_trees[2]), Some(10)); } #[test] diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index e7bfd5fd2..209ca4048 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -64,12 +64,14 @@ pub enum Delimiter { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Literal { pub text: SmolStr, + pub id: TokenId, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Punct { pub char: char, pub spacing: Spacing, + pub id: TokenId, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -- cgit v1.2.3