From 59295854f892b0a8f42a6fbc80b04d1f1c695828 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 13 Dec 2019 01:41:44 +0800 Subject: Add token id to delims --- crates/ra_hir_expand/src/quote.rs | 14 ++- crates/ra_mbe/src/mbe_expander/matcher.rs | 2 +- crates/ra_mbe/src/subtree_source.rs | 8 +- crates/ra_mbe/src/syntax_bridge.rs | 144 ++++++++++++++++++++---------- crates/ra_mbe/src/tests.rs | 16 ++-- crates/ra_tt/src/lib.rs | 16 ++-- 6 files changed, 135 insertions(+), 65 deletions(-) (limited to 'crates') diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs index bce38cc67..49155fe62 100644 --- a/crates/ra_hir_expand/src/quote.rs +++ b/crates/ra_hir_expand/src/quote.rs @@ -16,7 +16,10 @@ macro_rules! __quote { { let children = $crate::__quote!($($tt)*); let subtree = tt::Subtree { - delimiter: Some(tt::Delimiter::$delim), + delimiter: Some(tt::Delimiter { + kind: tt::DelimiterKind::$delim, + id: tt::TokenId::unspecified(), + }), token_trees: $crate::quote::IntoTt::to_tokens(children), }; subtree @@ -257,8 +260,13 @@ mod tests { let fields = fields.iter().map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone()).flatten(); - let list = - tt::Subtree { delimiter: Some(tt::Delimiter::Brace), token_trees: fields.collect() }; + let list = tt::Subtree { + delimiter: Some(tt::Delimiter { + kind: tt::DelimiterKind::Brace, + id: tt::TokenId::unspecified(), + }), + token_trees: fields.collect(), + }; let quoted = quote! { impl Clone for #struct_name { diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 3f5136478..c67ae4110 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -106,7 +106,7 @@ fn match_subtree( } Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?; - if lhs.delimiter != rhs.delimiter { + if lhs.delimiter.map(|it| it.kind) != rhs.delimiter.map(|it| it.kind) { bail!("mismatched delimiter") } let mut src = TtIter::new(rhs); diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 061e9f20b..5a03a372a 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -115,10 +115,10 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> { } fn convert_delim(d: Option, closing: bool) -> TtToken { - let (kinds, texts) = match d { - Some(tt::Delimiter::Parenthesis) => ([T!['('], T![')']], "()"), - Some(tt::Delimiter::Brace) => ([T!['{'], T!['}']], "{}"), - Some(tt::Delimiter::Bracket) => ([T!['['], T![']']], "[]"), + let (kinds, texts) = match d.map(|it| it.kind) { + Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"), + Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"), + Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"), None => ([L_DOLLAR, R_DOLLAR], ""), }; diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 8f65ff125..a85bb058b 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -5,6 +5,7 @@ use ra_syntax::{ ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode, SyntaxTreeBuilder, TextRange, TextUnit, T, }; +use rustc_hash::FxHashMap; use std::iter::successors; use tt::buffer::{Cursor, TokenBuffer}; @@ -83,6 +84,15 @@ impl TokenMap { fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { self.entries.push((token_id, relative_range)); } + + fn insert_delim( + &mut self, + _token_id: tt::TokenId, + _open_relative_range: TextRange, + _close_relative_range: TextRange, + ) { + // FIXME: Add entries for delimiter + } } /// Returns the textual content of a doc comment block as a quoted string @@ -121,7 +131,10 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option (Some(tt::Delimiter::Parenthesis), true), - (T!['{'], T!['}']) => (Some(tt::Delimiter::Brace), true), - (T!['['], T![']']) => (Some(tt::Delimiter::Bracket), true), + let (delimiter_kind, skip_first) = match (first_child.kind(), last_child.kind()) { + (T!['('], T![')']) => (Some(tt::DelimiterKind::Parenthesis), true), + (T!['{'], T!['}']) => (Some(tt::DelimiterKind::Brace), true), + (T!['['], T![']']) => (Some(tt::DelimiterKind::Bracket), true), _ => (None, false), }; + let delimiter = delimiter_kind.map(|kind| tt::Delimiter { + kind, + id: self.alloc_delim(first_child.text_range(), last_child.text_range()), + }); let mut token_trees = Vec::new(); let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); @@ -232,25 +249,31 @@ impl Convertor { .push(tt::Leaf::from(tt::Punct { char, spacing, id }).into()); } } else { - let child: tt::TokenTree = if token.kind() == T![true] - || token.kind() == T![false] - { - let id = self.alloc(token.text_range()); - tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() - } else if token.kind().is_keyword() - || token.kind() == IDENT - || token.kind() == LIFETIME - { - let id = self.alloc(token.text_range()); - let text = token.text().clone(); - tt::Leaf::from(tt::Ident { text, id }).into() - } else if token.kind().is_literal() { - let id = self.alloc(token.text_range()); - tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() - } else { - return None; + let child: tt::Leaf = match token.kind() { + T![true] | T![false] => { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Literal { text, id }.into() + } + IDENT | LIFETIME => { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Ident { text, id }.into() + } + k if k.is_keyword() => { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Ident { text, id }.into() + } + k if k.is_literal() => { + let id = self.alloc(token.text_range()); + let text = token.text().clone(); + tt::Literal { text, id }.into() + } + _ => return None, }; - token_trees.push(child); + + token_trees.push(child.into()); } } NodeOrToken::Node(node) => { @@ -275,11 +298,26 @@ impl Convertor { self.map.insert(token_id, relative_range); token_id } + + fn alloc_delim( + &mut self, + open_abs_range: TextRange, + close_abs_range: TextRange, + ) -> tt::TokenId { + let open_relative_range = open_abs_range - self.global_offset; + let close_relative_range = close_abs_range - self.global_offset; + let token_id = tt::TokenId(self.next_id); + self.next_id += 1; + + self.map.insert_delim(token_id, open_relative_range, close_relative_range); + token_id + } } struct TtTreeSink<'a> { buf: String, cursor: Cursor<'a>, + open_delims: FxHashMap, text_pos: TextUnit, inner: SyntaxTreeBuilder, token_map: TokenMap, @@ -294,6 +332,7 @@ impl<'a> TtTreeSink<'a> { TtTreeSink { buf: String::new(), cursor, + open_delims: FxHashMap::default(), text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), @@ -307,10 +346,10 @@ impl<'a> TtTreeSink<'a> { } fn delim_to_str(d: Option, closing: bool) -> SmolStr { - let texts = match d { - Some(tt::Delimiter::Parenthesis) => "()", - Some(tt::Delimiter::Brace) => "{}", - Some(tt::Delimiter::Bracket) => "[]", + let texts = match d.map(|it| it.kind) { + Some(tt::DelimiterKind::Parenthesis) => "()", + Some(tt::DelimiterKind::Brace) => "{}", + Some(tt::DelimiterKind::Bracket) => "[]", None => return "".into(), }; @@ -331,34 +370,49 @@ impl<'a> TreeSink for TtTreeSink<'a> { break; } - match self.cursor.token_tree() { + let text: Option = match self.cursor.token_tree() { Some(tt::TokenTree::Leaf(leaf)) => { // Mark the range if needed - if let tt::Leaf::Ident(ident) = leaf { - if kind == IDENT { - let range = - TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text)); - self.token_map.insert(ident.id, range); - } - } - + let id = match leaf { + tt::Leaf::Ident(ident) => ident.id, + tt::Leaf::Punct(punct) => punct.id, + tt::Leaf::Literal(lit) => lit.id, + }; + let text = SmolStr::new(format!("{}", leaf)); + let range = TextRange::offset_len(self.text_pos, TextUnit::of_str(&text)); + self.token_map.insert(id, range); self.cursor = self.cursor.bump(); - self.buf += &format!("{}", leaf); + Some(text) } Some(tt::TokenTree::Subtree(subtree)) => { self.cursor = self.cursor.subtree().unwrap(); - self.buf += &delim_to_str(subtree.delimiter, false); - } - None => { - if let Some(parent) = self.cursor.end() { - self.cursor = self.cursor.bump(); - self.buf += &delim_to_str(parent.delimiter, true); + if let Some(id) = subtree.delimiter.map(|it| it.id) { + self.open_delims.insert(id, self.text_pos); } + Some(delim_to_str(subtree.delimiter, false)) } + None => self.cursor.end().and_then(|parent| { + self.cursor = self.cursor.bump(); + if let Some(id) = parent.delimiter.map(|it| it.id) { + if let Some(open_delim) = self.open_delims.get(&id) { + let open_range = + TextRange::offset_len(*open_delim, TextUnit::from_usize(1)); + let close_range = + TextRange::offset_len(self.text_pos, TextUnit::from_usize(1)); + self.token_map.insert_delim(id, open_range, close_range); + } + } + + Some(delim_to_str(parent.delimiter, true)) + }), }; + + if let Some(text) = text { + self.buf += &text; + self.text_pos += TextUnit::of_str(&text); + } } - self.text_pos += TextUnit::of_str(&self.buf); let text = SmolStr::new(self.buf.as_str()); self.buf.clear(); self.inner.token(kind, text); @@ -504,7 +558,7 @@ mod tests { let token_tree = ast::TokenTree::cast(token_tree).unwrap(); let tt = ast_to_token_tree(&token_tree).unwrap().0; - assert_eq!(tt.delimiter, Some(tt::Delimiter::Brace)); + assert_eq!(tt.delimiter.map(|it| it.kind), Some(tt::DelimiterKind::Brace)); } #[test] diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 70e65bc74..6bcfedcac 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -77,13 +77,15 @@ macro_rules! foobar { } assert_eq!(expansion.token_trees.len(), 3); - // ($e:ident) => { foo bar $e } - // 0123 45 6 7 89 - assert_eq!(get_id(&expansion.token_trees[0]), Some(6)); - assert_eq!(get_id(&expansion.token_trees[1]), Some(7)); - - // So baz should be 10 - assert_eq!(get_id(&expansion.token_trees[2]), Some(10)); + // {($e:ident) => { foo bar $e }} + // 012345 67 8 9 T 12 + assert_eq!(get_id(&expansion.token_trees[0]), Some(9)); + assert_eq!(get_id(&expansion.token_trees[1]), Some(10)); + + // The input args of macro call include parentheses: + // (baz) + // So baz should be 12+1+1 + assert_eq!(get_id(&expansion.token_trees[2]), Some(14)); } #[test] diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index 209ca4048..73d8395a8 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -55,7 +55,13 @@ pub struct Subtree { } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum Delimiter { +pub struct Delimiter { + pub id: TokenId, + pub kind: DelimiterKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DelimiterKind { Parenthesis, Brace, Bracket, @@ -97,10 +103,10 @@ impl fmt::Display for TokenTree { impl fmt::Display for Subtree { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let (l, r) = match self.delimiter { - Some(Delimiter::Parenthesis) => ("(", ")"), - Some(Delimiter::Brace) => ("{", "}"), - Some(Delimiter::Bracket) => ("[", "]"), + let (l, r) = match self.delimiter.map(|it| it.kind) { + Some(DelimiterKind::Parenthesis) => ("(", ")"), + Some(DelimiterKind::Brace) => ("{", "}"), + Some(DelimiterKind::Bracket) => ("[", "]"), None => ("", ""), }; f.write_str(l)?; -- cgit v1.2.3