From aceb9d7fb0809ccf364514d9177342edea144c59 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Thu, 12 Dec 2019 21:47:54 +0800 Subject: Add token ids for all tt::Leaf --- crates/ra_hir_expand/src/builtin_derive.rs | 28 ++++++++++++++--- crates/ra_hir_expand/src/quote.rs | 13 +++++--- crates/ra_mbe/src/mbe_expander/transcriber.rs | 7 ++++- crates/ra_mbe/src/syntax_bridge.rs | 45 ++++++++++++++++----------- crates/ra_mbe/src/tests.rs | 10 +++--- crates/ra_tt/src/lib.rs | 2 ++ 6 files changed, 72 insertions(+), 33 deletions(-) (limited to 'crates') diff --git a/crates/ra_hir_expand/src/builtin_derive.rs b/crates/ra_hir_expand/src/builtin_derive.rs index b26441253..62c60e336 100644 --- a/crates/ra_hir_expand/src/builtin_derive.rs +++ b/crates/ra_hir_expand/src/builtin_derive.rs @@ -97,11 +97,24 @@ fn parse_adt(tt: &tt::Subtree) -> Result { fn make_type_args(n: usize, bound: Vec) -> Vec { let mut result = Vec::::new(); - result.push(tt::Leaf::Punct(tt::Punct { char: '<', spacing: tt::Spacing::Alone }).into()); + result.push( + tt::Leaf::Punct(tt::Punct { + char: '<', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), + ); for i in 0..n { if i > 0 { - result - .push(tt::Leaf::Punct(tt::Punct { char: ',', spacing: tt::Spacing::Alone }).into()); + result.push( + tt::Leaf::Punct(tt::Punct { + char: ',', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), + ); } result.push( tt::Leaf::Ident(tt::Ident { @@ -112,7 +125,14 @@ fn make_type_args(n: usize, bound: Vec) -> Vec { ); result.extend(bound.iter().cloned()); } - result.push(tt::Leaf::Punct(tt::Punct { char: '>', spacing: tt::Spacing::Alone }).into()); + result.push( + tt::Leaf::Punct(tt::Punct { + char: '>', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), + ); result } diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs index aa8a5f23f..bce38cc67 100644 --- a/crates/ra_hir_expand/src/quote.rs +++ b/crates/ra_hir_expand/src/quote.rs @@ -29,6 +29,7 @@ macro_rules! __quote { tt::Leaf::Punct(tt::Punct { char: $first, spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), }).into() ] } @@ -40,10 +41,12 @@ macro_rules! __quote { tt::Leaf::Punct(tt::Punct { char: $first, spacing: tt::Spacing::Joint, + id: tt::TokenId::unspecified(), }).into(), tt::Leaf::Punct(tt::Punct { char: $sec, spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), }).into() ] } @@ -179,15 +182,15 @@ macro_rules! impl_to_to_tokentrees { } impl_to_to_tokentrees! { - u32 => self { tt::Literal{text: self.to_string().into()} }; - usize => self { tt::Literal{text: self.to_string().into()}}; - i32 => self { tt::Literal{text: self.to_string().into()}}; + u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; + usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()}}; + i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()}}; tt::Leaf => self { self }; tt::Literal => self { self }; tt::Ident => self { self }; tt::Punct => self { self }; - &str => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into()}}; - String => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into()}} + &str => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into(), id: tt::TokenId::unspecified()}}; + String => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into(), id: tt::TokenId::unspecified()}} } #[cfg(test)] diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index f7636db11..eda66cd50 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs @@ -108,7 +108,12 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result let tt = tt::Subtree { delimiter: None, token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(), + tt::Leaf::from(tt::Punct { + char: '$', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) .into(), ], diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index b8e2cfc1d..8f65ff125 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -136,11 +136,15 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option tt::TokenTree { - tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone })) + tt::TokenTree::from(tt::Leaf::from(tt::Punct { + char: c, + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + })) } fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree { - let lit = tt::Literal { text: doc_comment_text(comment) }; + let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() }; tt::TokenTree::from(tt::Leaf::from(lit)) } @@ -223,24 +227,29 @@ impl Convertor { .take(token.text().len() - 1) .chain(std::iter::once(last_spacing)); for (char, spacing) in token.text().chars().zip(spacing_iter) { - token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into()); + let id = self.alloc(token.text_range()); + token_trees + .push(tt::Leaf::from(tt::Punct { char, spacing, id }).into()); } } else { - let child: tt::TokenTree = - if token.kind() == T![true] || token.kind() == T![false] { - tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() - } else if token.kind().is_keyword() - || token.kind() == IDENT - || token.kind() == LIFETIME - { - let id = self.alloc(token.text_range()); - let text = token.text().clone(); - tt::Leaf::from(tt::Ident { text, id }).into() - } else if token.kind().is_literal() { - tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() - } else { - return None; - }; + let child: tt::TokenTree = if token.kind() == T![true] + || token.kind() == T![false] + { + let id = self.alloc(token.text_range()); + tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() + } else if token.kind().is_keyword() + || token.kind() == IDENT + || token.kind() == LIFETIME + { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Leaf::from(tt::Ident { text, id }).into() + } else if token.kind().is_literal() { + let id = self.alloc(token.text_range()); + tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() + } else { + return None; + }; token_trees.push(child); } } diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 148cc2625..70e65bc74 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -78,12 +78,12 @@ macro_rules! foobar { assert_eq!(expansion.token_trees.len(), 3); // ($e:ident) => { foo bar $e } - // 0 1 2 3 4 - assert_eq!(get_id(&expansion.token_trees[0]), Some(2)); - assert_eq!(get_id(&expansion.token_trees[1]), Some(3)); + // 0123 45 6 7 89 + assert_eq!(get_id(&expansion.token_trees[0]), Some(6)); + assert_eq!(get_id(&expansion.token_trees[1]), Some(7)); - // So baz should be 5 - assert_eq!(get_id(&expansion.token_trees[2]), Some(5)); + // So baz should be 10 + assert_eq!(get_id(&expansion.token_trees[2]), Some(10)); } #[test] diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index e7bfd5fd2..209ca4048 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -64,12 +64,14 @@ pub enum Delimiter { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Literal { pub text: SmolStr, + pub id: TokenId, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Punct { pub char: char, pub spacing: Spacing, + pub id: TokenId, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -- cgit v1.2.3 From 59295854f892b0a8f42a6fbc80b04d1f1c695828 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 13 Dec 2019 01:41:44 +0800 Subject: Add token id to delims --- crates/ra_hir_expand/src/quote.rs | 14 ++- crates/ra_mbe/src/mbe_expander/matcher.rs | 2 +- crates/ra_mbe/src/subtree_source.rs | 8 +- crates/ra_mbe/src/syntax_bridge.rs | 144 ++++++++++++++++++++---------- crates/ra_mbe/src/tests.rs | 16 ++-- crates/ra_tt/src/lib.rs | 16 ++-- 6 files changed, 135 insertions(+), 65 deletions(-) (limited to 'crates') diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs index bce38cc67..49155fe62 100644 --- a/crates/ra_hir_expand/src/quote.rs +++ b/crates/ra_hir_expand/src/quote.rs @@ -16,7 +16,10 @@ macro_rules! __quote { { let children = $crate::__quote!($($tt)*); let subtree = tt::Subtree { - delimiter: Some(tt::Delimiter::$delim), + delimiter: Some(tt::Delimiter { + kind: tt::DelimiterKind::$delim, + id: tt::TokenId::unspecified(), + }), token_trees: $crate::quote::IntoTt::to_tokens(children), }; subtree @@ -257,8 +260,13 @@ mod tests { let fields = fields.iter().map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone()).flatten(); - let list = - tt::Subtree { delimiter: Some(tt::Delimiter::Brace), token_trees: fields.collect() }; + let list = tt::Subtree { + delimiter: Some(tt::Delimiter { + kind: tt::DelimiterKind::Brace, + id: tt::TokenId::unspecified(), + }), + token_trees: fields.collect(), + }; let quoted = quote! { impl Clone for #struct_name { diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 3f5136478..c67ae4110 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -106,7 +106,7 @@ fn match_subtree( } Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?; - if lhs.delimiter != rhs.delimiter { + if lhs.delimiter.map(|it| it.kind) != rhs.delimiter.map(|it| it.kind) { bail!("mismatched delimiter") } let mut src = TtIter::new(rhs); diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 061e9f20b..5a03a372a 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -115,10 +115,10 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> { } fn convert_delim(d: Option, closing: bool) -> TtToken { - let (kinds, texts) = match d { - Some(tt::Delimiter::Parenthesis) => ([T!['('], T![')']], "()"), - Some(tt::Delimiter::Brace) => ([T!['{'], T!['}']], "{}"), - Some(tt::Delimiter::Bracket) => ([T!['['], T![']']], "[]"), + let (kinds, texts) = match d.map(|it| it.kind) { + Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"), + Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"), + Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"), None => ([L_DOLLAR, R_DOLLAR], ""), }; diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 8f65ff125..a85bb058b 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -5,6 +5,7 @@ use ra_syntax::{ ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode, SyntaxTreeBuilder, TextRange, TextUnit, T, }; +use rustc_hash::FxHashMap; use std::iter::successors; use tt::buffer::{Cursor, TokenBuffer}; @@ -83,6 +84,15 @@ impl TokenMap { fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { self.entries.push((token_id, relative_range)); } + + fn insert_delim( + &mut self, + _token_id: tt::TokenId, + _open_relative_range: TextRange, + _close_relative_range: TextRange, + ) { + // FIXME: Add entries for delimiter + } } /// Returns the textual content of a doc comment block as a quoted string @@ -121,7 +131,10 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option (Some(tt::Delimiter::Parenthesis), true), - (T!['{'], T!['}']) => (Some(tt::Delimiter::Brace), true), - (T!['['], T![']']) => (Some(tt::Delimiter::Bracket), true), + let (delimiter_kind, skip_first) = match (first_child.kind(), last_child.kind()) { + (T!['('], T![')']) => (Some(tt::DelimiterKind::Parenthesis), true), + (T!['{'], T!['}']) => (Some(tt::DelimiterKind::Brace), true), + (T!['['], T![']']) => (Some(tt::DelimiterKind::Bracket), true), _ => (None, false), }; + let delimiter = delimiter_kind.map(|kind| tt::Delimiter { + kind, + id: self.alloc_delim(first_child.text_range(), last_child.text_range()), + }); let mut token_trees = Vec::new(); let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); @@ -232,25 +249,31 @@ impl Convertor { .push(tt::Leaf::from(tt::Punct { char, spacing, id }).into()); } } else { - let child: tt::TokenTree = if token.kind() == T![true] - || token.kind() == T![false] - { - let id = self.alloc(token.text_range()); - tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() - } else if token.kind().is_keyword() - || token.kind() == IDENT - || token.kind() == LIFETIME - { - let id = self.alloc(token.text_range()); - let text = token.text().clone(); - tt::Leaf::from(tt::Ident { text, id }).into() - } else if token.kind().is_literal() { - let id = self.alloc(token.text_range()); - tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() - } else { - return None; + let child: tt::Leaf = match token.kind() { + T![true] | T![false] => { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Literal { text, id }.into() + } + IDENT | LIFETIME => { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Ident { text, id }.into() + } + k if k.is_keyword() => { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Ident { text, id }.into() + } + k if k.is_literal() => { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Literal { text, id }.into() + } + _ => return None, }; - token_trees.push(child); + + token_trees.push(child.into()); } } NodeOrToken::Node(node) => { @@ -275,11 +298,26 @@ impl Convertor { self.map.insert(token_id, relative_range); token_id } + + fn alloc_delim( + &mut self, + open_abs_range: TextRange, + close_abs_range: TextRange, + ) -> tt::TokenId { + let open_relative_range = open_abs_range - self.global_offset; + let close_relative_range = close_abs_range - self.global_offset; + let token_id = tt::TokenId(self.next_id); + self.next_id += 1; + + self.map.insert_delim(token_id, open_relative_range, close_relative_range); + token_id + } } struct TtTreeSink<'a> { buf: String, cursor: Cursor<'a>, + open_delims: FxHashMap, text_pos: TextUnit, inner: SyntaxTreeBuilder, token_map: TokenMap, @@ -294,6 +332,7 @@ impl<'a> TtTreeSink<'a> { TtTreeSink { buf: String::new(), cursor, + open_delims: FxHashMap::default(), text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), @@ -307,10 +346,10 @@ impl<'a> TtTreeSink<'a> { } fn delim_to_str(d: Option, closing: bool) -> SmolStr { - let texts = match d { - Some(tt::Delimiter::Parenthesis) => "()", - Some(tt::Delimiter::Brace) => "{}", - Some(tt::Delimiter::Bracket) => "[]", + let texts = match d.map(|it| it.kind) { + Some(tt::DelimiterKind::Parenthesis) => "()", + Some(tt::DelimiterKind::Brace) => "{}", + Some(tt::DelimiterKind::Bracket) => "[]", None => return "".into(), }; @@ -331,34 +370,49 @@ impl<'a> TreeSink for TtTreeSink<'a> { break; } - match self.cursor.token_tree() { + let text: Option = match self.cursor.token_tree() { Some(tt::TokenTree::Leaf(leaf)) => { // Mark the range if needed - if let tt::Leaf::Ident(ident) = leaf { - if kind == IDENT { - let range = - TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text)); - self.token_map.insert(ident.id, range); - } - } - + let id = match leaf { + tt::Leaf::Ident(ident) => ident.id, + tt::Leaf::Punct(punct) => punct.id, + tt::Leaf::Literal(lit) => lit.id, + }; + let text = SmolStr::new(format!("{}", leaf)); + let range = TextRange::offset_len(self.text_pos, TextUnit::of_str(&text)); + self.token_map.insert(id, range); self.cursor = self.cursor.bump(); - self.buf += &format!("{}", leaf); + Some(text) } Some(tt::TokenTree::Subtree(subtree)) => { self.cursor = self.cursor.subtree().unwrap(); - self.buf += &delim_to_str(subtree.delimiter, false); - } - None => { - if let Some(parent) = self.cursor.end() { - self.cursor = self.cursor.bump(); - self.buf += &delim_to_str(parent.delimiter, true); + if let Some(id) = subtree.delimiter.map(|it| it.id) { + self.open_delims.insert(id, self.text_pos); } + Some(delim_to_str(subtree.delimiter, false)) } + None => self.cursor.end().and_then(|parent| { + self.cursor = self.cursor.bump(); + if let Some(id) = parent.delimiter.map(|it| it.id) { + if let Some(open_delim) = self.open_delims.get(&id) { + let open_range = + TextRange::offset_len(*open_delim, TextUnit::from_usize(1)); + let close_range = + TextRange::offset_len(self.text_pos, TextUnit::from_usize(1)); + self.token_map.insert_delim(id, open_range, close_range); + } + } + + Some(delim_to_str(parent.delimiter, true)) + }), }; + + if let Some(text) = text { + self.buf += &text; + self.text_pos += TextUnit::of_str(&text); + } } - self.text_pos += TextUnit::of_str(&self.buf); let text = SmolStr::new(self.buf.as_str()); self.buf.clear(); self.inner.token(kind, text); @@ -504,7 +558,7 @@ mod tests { let token_tree = ast::TokenTree::cast(token_tree).unwrap(); let tt = ast_to_token_tree(&token_tree).unwrap().0; - assert_eq!(tt.delimiter, Some(tt::Delimiter::Brace)); + assert_eq!(tt.delimiter.map(|it| it.kind), Some(tt::DelimiterKind::Brace)); } #[test] diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 70e65bc74..6bcfedcac 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -77,13 +77,15 @@ macro_rules! foobar { } assert_eq!(expansion.token_trees.len(), 3); - // ($e:ident) => { foo bar $e } - // 0123 45 6 7 89 - assert_eq!(get_id(&expansion.token_trees[0]), Some(6)); - assert_eq!(get_id(&expansion.token_trees[1]), Some(7)); - - // So baz should be 10 - assert_eq!(get_id(&expansion.token_trees[2]), Some(10)); + // {($e:ident) => { foo bar $e }} + // 012345 67 8 9 T 12 + assert_eq!(get_id(&expansion.token_trees[0]), Some(9)); + assert_eq!(get_id(&expansion.token_trees[1]), Some(10)); + + // The input args of macro call include parentheses: + // (baz) + // So baz should be 12+1+1 + assert_eq!(get_id(&expansion.token_trees[2]), Some(14)); } #[test] diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index 209ca4048..73d8395a8 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -55,7 +55,13 @@ pub struct Subtree { } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum Delimiter { +pub struct Delimiter { + pub id: TokenId, + pub kind: DelimiterKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DelimiterKind { Parenthesis, Brace, Bracket, @@ -97,10 +103,10 @@ impl fmt::Display for TokenTree { impl fmt::Display for Subtree { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let (l, r) = match self.delimiter { - Some(Delimiter::Parenthesis) => ("(", ")"), - Some(Delimiter::Brace) => ("{", "}"), - Some(Delimiter::Bracket) => ("[", "]"), + let (l, r) = match self.delimiter.map(|it| it.kind) { + Some(DelimiterKind::Parenthesis) => ("(", ")"), + Some(DelimiterKind::Brace) => ("{", "}"), + Some(DelimiterKind::Bracket) => ("[", "]"), None => ("", ""), }; f.write_str(l)?; -- cgit v1.2.3 From 320416d7561e9926ecbbc392cc2efd48740610e0 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 13 Dec 2019 23:55:51 +0800 Subject: Add TokenTextRange --- crates/ra_hir_expand/src/lib.rs | 4 ++-- crates/ra_mbe/src/syntax_bridge.rs | 41 ++++++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 10 deletions(-) (limited to 'crates') diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index cb4e1950b..720a29ea5 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -227,7 +227,7 @@ impl ExpansionInfo { let token_id = self.macro_arg.1.token_by_range(range)?; let token_id = self.macro_def.0.map_id_down(token_id); - let range = self.exp_map.range_by_token(token_id)?; + let range = self.exp_map.range_by_token(token_id)?.range(token.value.kind())?; let token = algo::find_covering_element(&self.expanded.value, range).into_token()?; @@ -248,7 +248,7 @@ impl ExpansionInfo { } }; - let range = token_map.range_by_token(token_id)?; + let range = token_map.range_by_token(token_id)?.range(token.value.kind())?; let token = algo::find_covering_element(&tt.value, range + tt.value.text_range().start()) .into_token()?; Some((tt.with_value(token), origin)) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index a85bb058b..44a51b7a5 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -12,11 +12,30 @@ use tt::buffer::{Cursor, TokenBuffer}; use crate::subtree_source::SubtreeTokenSource; use crate::ExpandError; +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum TokenTextRange { + Token(TextRange), + Delimiter(TextRange, TextRange), +} + +impl TokenTextRange { + pub fn range(self, kind: SyntaxKind) -> Option { + match self { + TokenTextRange::Token(it) => Some(it), + TokenTextRange::Delimiter(open, close) => match kind { + T!['{'] | T!['('] | T!['['] => Some(open), + T!['}'] | T![')'] | T![']'] => Some(close), + _ => None, + }, + } + } +} + /// Maps `tt::TokenId` to the relative range of the original token. #[derive(Debug, PartialEq, Eq, Default)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. - entries: Vec<(tt::TokenId, TextRange)>, + entries: Vec<(tt::TokenId, TokenTextRange)>, } /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro @@ -72,26 +91,32 @@ pub fn token_tree_to_syntax_node( impl TokenMap { pub fn token_by_range(&self, relative_range: TextRange) -> Option { - let &(token_id, _) = self.entries.iter().find(|(_, range)| *range == relative_range)?; + let &(token_id, _) = self.entries.iter().find(|(_, range)| match range { + TokenTextRange::Token(it) => *it == relative_range, + TokenTextRange::Delimiter(open, close) => { + *open == relative_range || *close == relative_range + } + })?; Some(token_id) } - pub fn range_by_token(&self, token_id: tt::TokenId) -> Option { + pub fn range_by_token(&self, token_id: tt::TokenId) -> Option { let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?; Some(range) } fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { - self.entries.push((token_id, relative_range)); + self.entries.push((token_id, TokenTextRange::Token(relative_range))); } fn insert_delim( &mut self, - _token_id: tt::TokenId, - _open_relative_range: TextRange, - _close_relative_range: TextRange, + token_id: tt::TokenId, + open_relative_range: TextRange, + close_relative_range: TextRange, ) { - // FIXME: Add entries for delimiter + self.entries + .push((token_id, TokenTextRange::Delimiter(open_relative_range, close_relative_range))); } } -- cgit v1.2.3 From 325532d11960733bebdd38b19a6f33a891872803 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 14 Dec 2019 03:37:04 +0800 Subject: Fix shift id for delim and other tokens --- crates/ra_mbe/src/lib.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index ce2deadf6..45dad2d10 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -67,7 +67,15 @@ impl Shift { .token_trees .iter() .filter_map(|tt| match tt { - tt::TokenTree::Subtree(subtree) => max_id(subtree), + tt::TokenTree::Subtree(subtree) => { + let tree_id = max_id(subtree); + match subtree.delimiter { + Some(it) if it.id != tt::TokenId::unspecified() => { + Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0))) + } + _ => tree_id, + } + } tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) if ident.id != tt::TokenId::unspecified() => { @@ -85,9 +93,13 @@ impl Shift { match t { tt::TokenTree::Leaf(leaf) => match leaf { tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id), - _ => (), + tt::Leaf::Punct(punct) => punct.id = self.shift(punct.id), + tt::Leaf::Literal(lit) => lit.id = self.shift(lit.id), }, - tt::TokenTree::Subtree(tt) => self.shift_all(tt), + tt::TokenTree::Subtree(tt) => { + tt.delimiter.as_mut().map(|it: &mut Delimiter| it.id = self.shift(it.id)); + self.shift_all(tt) + } } } } -- cgit v1.2.3 From e16f3a5ee2f7df3f42827f3f279b5ed6774bde8e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 14 Dec 2019 03:39:15 +0800 Subject: Add test for token map --- crates/ra_mbe/src/tests.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'crates') diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 6bcfedcac..ae7f3dfd4 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -88,6 +88,32 @@ macro_rules! foobar { assert_eq!(get_id(&expansion.token_trees[2]), Some(14)); } +#[test] +fn test_token_map() { + use ra_parser::SyntaxKind::*; + use ra_syntax::T; + + let macro_definition = r#" +macro_rules! foobar { + ($e:ident) => { fn $e() {} } +} +"#; + let rules = create_rules(macro_definition); + let (expansion, (token_map, content)) = expand_and_map(&rules, "foobar!(baz);"); + + let get_text = |id, kind| -> String { + content[token_map.range_by_token(id).unwrap().range(kind).unwrap()].to_string() + }; + + assert_eq!(expansion.token_trees.len(), 4); + // {($e:ident) => { fn $e() {} }} + // 012345 67 8 9 T12 3 + + assert_eq!(get_text(tt::TokenId(9), IDENT), "fn"); + assert_eq!(get_text(tt::TokenId(12), T!['(']), "("); + assert_eq!(get_text(tt::TokenId(13), T!['{']), "{"); +} + #[test] fn test_convert_tt() { let macro_definition = r#" @@ -1443,6 +1469,23 @@ pub(crate) fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree { rules.expand(&invocation_tt).unwrap() } +pub(crate) fn expand_and_map( + rules: &MacroRules, + invocation: &str, +) -> (tt::Subtree, (TokenMap, String)) { + let source_file = ast::SourceFile::parse(invocation).ok().unwrap(); + let macro_invocation = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + + let (invocation_tt, _) = ast_to_token_tree(¯o_invocation.token_tree().unwrap()).unwrap(); + let expanded = rules.expand(&invocation_tt).unwrap(); + + let (node, expanded_token_tree) = + token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap(); + + (expanded, (expanded_token_tree, node.syntax_node().to_string())) +} + pub(crate) enum MacroKind { Items, Stmts, -- cgit v1.2.3 From 2ea1cfd7804dd57d63196e71f66c22e56cfd79a8 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Wed, 18 Dec 2019 11:36:10 +0800 Subject: Rename range to by_kind --- crates/ra_hir_expand/src/lib.rs | 4 ++-- crates/ra_mbe/src/syntax_bridge.rs | 2 +- crates/ra_mbe/src/tests.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'crates') diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index 720a29ea5..2fa5d5140 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -227,7 +227,7 @@ impl ExpansionInfo { let token_id = self.macro_arg.1.token_by_range(range)?; let token_id = self.macro_def.0.map_id_down(token_id); - let range = self.exp_map.range_by_token(token_id)?.range(token.value.kind())?; + let range = self.exp_map.range_by_token(token_id)?.by_kind(token.value.kind())?; let token = algo::find_covering_element(&self.expanded.value, range).into_token()?; @@ -248,7 +248,7 @@ impl ExpansionInfo { } }; - let range = token_map.range_by_token(token_id)?.range(token.value.kind())?; + let range = token_map.range_by_token(token_id)?.by_kind(token.value.kind())?; let token = algo::find_covering_element(&tt.value, range + tt.value.text_range().start()) .into_token()?; Some((tt.with_value(token), origin)) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 44a51b7a5..d585d57af 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -19,7 +19,7 @@ pub enum TokenTextRange { } impl TokenTextRange { - pub fn range(self, kind: SyntaxKind) -> Option { + pub fn by_kind(self, kind: SyntaxKind) -> Option { match self { TokenTextRange::Token(it) => Some(it), TokenTextRange::Delimiter(open, close) => match kind { diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index ae7f3dfd4..ff225f0db 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -102,7 +102,7 @@ macro_rules! foobar { let (expansion, (token_map, content)) = expand_and_map(&rules, "foobar!(baz);"); let get_text = |id, kind| -> String { - content[token_map.range_by_token(id).unwrap().range(kind).unwrap()].to_string() + content[token_map.range_by_token(id).unwrap().by_kind(kind).unwrap()].to_string() }; assert_eq!(expansion.token_trees.len(), 4); -- cgit v1.2.3 From 41544a40883874553f570e2999bf56d172bd6246 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Wed, 18 Dec 2019 11:47:26 +0800 Subject: Refactoring --- crates/ra_mbe/src/mbe_expander/matcher.rs | 2 +- crates/ra_mbe/src/subtree_source.rs | 8 +-- crates/ra_mbe/src/syntax_bridge.rs | 92 +++++++++++++++---------------- crates/ra_tt/src/lib.rs | 6 +- 4 files changed, 55 insertions(+), 53 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index c67ae4110..e36b5a412 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -106,7 +106,7 @@ fn match_subtree( } Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?; - if lhs.delimiter.map(|it| it.kind) != rhs.delimiter.map(|it| it.kind) { + if lhs.delimiter_kind() != rhs.delimiter_kind() { bail!("mismatched delimiter") } let mut src = TtIter::new(rhs); diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 5a03a372a..b841c39d3 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -70,11 +70,11 @@ impl<'a> SubtreeTokenSource<'a> { } Some(tt::TokenTree::Subtree(subtree)) => { self.cached_cursor.set(cursor.subtree().unwrap()); - cached.push(Some(convert_delim(subtree.delimiter, false))); + cached.push(Some(convert_delim(subtree.delimiter_kind(), false))); } None => { if let Some(subtree) = cursor.end() { - cached.push(Some(convert_delim(subtree.delimiter, true))); + cached.push(Some(convert_delim(subtree.delimiter_kind(), true))); self.cached_cursor.set(cursor.bump()); } } @@ -114,8 +114,8 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> { } } -fn convert_delim(d: Option, closing: bool) -> TtToken { - let (kinds, texts) = match d.map(|it| it.kind) { +fn convert_delim(d: Option, closing: bool) -> TtToken { + let (kinds, texts) = match d { Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"), Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"), Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"), diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index d585d57af..2c60430d1 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -269,35 +269,33 @@ impl Convertor { .take(token.text().len() - 1) .chain(std::iter::once(last_spacing)); for (char, spacing) in token.text().chars().zip(spacing_iter) { - let id = self.alloc(token.text_range()); - token_trees - .push(tt::Leaf::from(tt::Punct { char, spacing, id }).into()); + token_trees.push( + tt::Leaf::from(tt::Punct { + char, + spacing, + id: self.alloc(token.text_range()), + }) + .into(), + ); } } else { + macro_rules! make_leaf { + ($i:ident) => { + tt::$i { + id: self.alloc(token.text_range()), + text: token.text().clone(), + } + .into() + }; + } + let child: tt::Leaf = match token.kind() { - T![true] | T![false] => { - let id = self.alloc(token.text_range()); - let text = token.text().clone(); - tt::Literal { text, id }.into() - } - IDENT | LIFETIME => { - let id = self.alloc(token.text_range()); - let text = token.text().clone(); - tt::Ident { text, id }.into() - } - k if k.is_keyword() => { - let id = self.alloc(token.text_range()); - let text = token.text().clone(); - tt::Ident { text, id }.into() - } - k if k.is_literal() => { - let id = self.alloc(token.text_range()); - let text = token.text().clone(); - tt::Literal { text, id }.into() - } + T![true] | T![false] => make_leaf!(Literal), + IDENT | LIFETIME => make_leaf!(Ident), + k if k.is_keyword() => make_leaf!(Ident), + k if k.is_literal() => make_leaf!(Literal), _ => return None, }; - token_trees.push(child.into()); } } @@ -370,8 +368,8 @@ impl<'a> TtTreeSink<'a> { } } -fn delim_to_str(d: Option, closing: bool) -> SmolStr { - let texts = match d.map(|it| it.kind) { +fn delim_to_str(d: Option, closing: bool) -> SmolStr { + let texts = match d { Some(tt::DelimiterKind::Parenthesis) => "()", Some(tt::DelimiterKind::Brace) => "{}", Some(tt::DelimiterKind::Bracket) => "[]", @@ -395,7 +393,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { break; } - let text: Option = match self.cursor.token_tree() { + let text: SmolStr = match self.cursor.token_tree() { Some(tt::TokenTree::Leaf(leaf)) => { // Mark the range if needed let id = match leaf { @@ -407,35 +405,35 @@ impl<'a> TreeSink for TtTreeSink<'a> { let range = TextRange::offset_len(self.text_pos, TextUnit::of_str(&text)); self.token_map.insert(id, range); self.cursor = self.cursor.bump(); - Some(text) + text } Some(tt::TokenTree::Subtree(subtree)) => { self.cursor = self.cursor.subtree().unwrap(); if let Some(id) = subtree.delimiter.map(|it| it.id) { self.open_delims.insert(id, self.text_pos); } - Some(delim_to_str(subtree.delimiter, false)) + delim_to_str(subtree.delimiter_kind(), false) } - None => self.cursor.end().and_then(|parent| { - self.cursor = self.cursor.bump(); - if let Some(id) = parent.delimiter.map(|it| it.id) { - if let Some(open_delim) = self.open_delims.get(&id) { - let open_range = - TextRange::offset_len(*open_delim, TextUnit::from_usize(1)); - let close_range = - TextRange::offset_len(self.text_pos, TextUnit::from_usize(1)); - self.token_map.insert_delim(id, open_range, close_range); + None => { + if let Some(parent) = self.cursor.end() { + self.cursor = self.cursor.bump(); + if let Some(id) = parent.delimiter.map(|it| it.id) { + if let Some(open_delim) = self.open_delims.get(&id) { + let open_range = + TextRange::offset_len(*open_delim, TextUnit::from_usize(1)); + let close_range = + TextRange::offset_len(self.text_pos, TextUnit::from_usize(1)); + self.token_map.insert_delim(id, open_range, close_range); + } } + delim_to_str(parent.delimiter_kind(), true) + } else { + continue; } - - Some(delim_to_str(parent.delimiter, true)) - }), + } }; - - if let Some(text) = text { - self.buf += &text; - self.text_pos += TextUnit::of_str(&text); - } + self.buf += &text; + self.text_pos += TextUnit::of_str(&text); } let text = SmolStr::new(self.buf.as_str()); @@ -583,7 +581,7 @@ mod tests { let token_tree = ast::TokenTree::cast(token_tree).unwrap(); let tt = ast_to_token_tree(&token_tree).unwrap().0; - assert_eq!(tt.delimiter.map(|it| it.kind), Some(tt::DelimiterKind::Brace)); + assert_eq!(tt.delimiter_kind(), Some(tt::DelimiterKind::Brace)); } #[test] diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index 73d8395a8..10f424aae 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -103,7 +103,7 @@ impl fmt::Display for TokenTree { impl fmt::Display for Subtree { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let (l, r) = match self.delimiter.map(|it| it.kind) { + let (l, r) = match self.delimiter_kind() { Some(DelimiterKind::Parenthesis) => ("(", ")"), Some(DelimiterKind::Brace) => ("{", "}"), Some(DelimiterKind::Bracket) => ("[", "]"), @@ -171,6 +171,10 @@ impl Subtree { self.token_trees.len() + children_count } + + pub fn delimiter_kind(&self) -> Option { + self.delimiter.map(|it| it.kind) + } } pub mod buffer; -- cgit v1.2.3