From 27c516970bea6bee9d6bca52ac9ad619412809ef Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Thu, 19 Mar 2020 20:40:25 +0800 Subject: Add test for delim bug --- crates/ra_mbe/src/syntax_bridge.rs | 103 +++++++++++++++++++++++++++++++++++++ crates/ra_mbe/src/tests.rs | 56 +++++++++++++++----- 2 files changed, 147 insertions(+), 12 deletions(-) (limited to 'crates/ra_mbe') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index fcb73fbc7..0678c37ee 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -267,6 +267,108 @@ struct RawConvertor<'a> { inner: std::slice::Iter<'a, Token>, } +trait SrcToken { + fn kind() -> SyntaxKind; +} + +trait TokenConvertor { + type Token : SrcToken; + + fn go(&mut self) -> Option { + let mut subtree = tt::Subtree::default(); + subtree.delimiter = None; + while self.peek().is_some() { + self.collect_leaf(&mut subtree.token_trees); + } + if subtree.token_trees.is_empty() { + return None; + } + if subtree.token_trees.len() == 1 { + if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] { + return Some(first.clone()); + } + } + Some(subtree) + } + + fn bump(&mut self) -> Option<(Self::Token, TextRange)>; + + fn peek(&self) -> Option; + + fn collect_leaf(&mut self, result: &mut Vec) { + let (token, range) = match self.bump() { + None => return, + Some(it) => it, + }; + + let k: SyntaxKind = token.kind(); + if k == COMMENT { + let node = doc_comment(&self.text[range]); + if let Some(tokens) = convert_doc_comment(&node) { + result.extend(tokens); + } + return; + } + + result.push(if k.is_punct() { + let delim = match k { + T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])), + T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])), + T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])), + _ => None, + }; + + if let Some((kind, closed)) = delim { + let mut subtree = tt::Subtree::default(); + let id = self.id_alloc.open_delim(range); + subtree.delimiter = Some(tt::Delimiter { kind, id }); + + while self.peek().map(|it| it.kind != closed).unwrap_or(false) { + self.collect_leaf(&mut subtree.token_trees); + } + let last_range = match self.bump() { + None => return, + Some(it) => it.1, + }; + self.id_alloc.close_delim(id, last_range); + subtree.into() + } else { + let spacing = match self.peek() { + Some(next) + if next.kind.is_trivia() + || next.kind == T!['['] + || next.kind == T!['{'] + || next.kind == T!['('] => + { + tt::Spacing::Alone + } + Some(next) if next.kind.is_punct() => tt::Spacing::Joint, + _ => tt::Spacing::Alone, + }; + let char = + self.text[range].chars().next().expect("Token from lexer must be single char"); + + tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc.alloc(range) }).into() + } + } else { + macro_rules! make_leaf { + ($i:ident) => { + tt::$i { id: self.id_alloc.alloc(range), text: self.text[range].into() }.into() + }; + } + let leaf: tt::Leaf = match k { + T![true] | T![false] => make_leaf!(Literal), + IDENT | LIFETIME => make_leaf!(Ident), + k if k.is_keyword() => make_leaf!(Ident), + k if k.is_literal() => make_leaf!(Literal), + _ => return, + }; + + leaf.into() + }); + } +} + impl RawConvertor<'_> { fn go(&mut self) -> Option { let mut subtree = tt::Subtree::default(); @@ -295,6 +397,7 @@ impl RawConvertor<'_> { fn peek(&self) -> Option { self.inner.as_slice().get(0).cloned() } + fn collect_leaf(&mut self, result: &mut Vec) { let (token, range) = match self.bump() { diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 44f381938..966af1d12 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -427,22 +427,28 @@ MACRO_ITEMS@[0; 40) ); } -#[test] -fn test_expand_literals_to_token_tree() { - fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { - if let tt::TokenTree::Subtree(subtree) = tt { - return &subtree; - } - unreachable!("It is not a subtree"); +fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { + if let tt::TokenTree::Subtree(subtree) = tt { + return &subtree; } + unreachable!("It is not a subtree"); +} +fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { + if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { + return lit; + } + unreachable!("It is not a literal"); +} - fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { - if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { - return lit; - } - unreachable!("It is not a literal"); +fn to_punct(tt: &tt::TokenTree) -> &tt::Punct { + if let tt::TokenTree::Leaf(tt::Leaf::Punct(lit)) = tt { + return lit; } + unreachable!("It is not a Punct"); +} +#[test] +fn test_expand_literals_to_token_tree() { let expansion = parse_macro( r#" macro_rules! literals { @@ -470,6 +476,22 @@ fn test_expand_literals_to_token_tree() { assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\""); } +#[test] +fn test_attr_to_token_tree() { + let expansion = parse_to_token_tree_by_syntax( + r#" + #[derive(Copy)] + struct Foo; + "#, + ); + + assert_eq!(to_punct(&expansion.token_trees[0]).char, '#'); + assert_eq!( + to_subtree(&expansion.token_trees[1]).delimiter_kind(), + Some(tt::DelimiterKind::Bracket) + ); +} + #[test] fn test_two_idents() { parse_macro( @@ -1517,6 +1539,16 @@ pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture { MacroFixture { rules } } +pub(crate) fn parse_to_token_tree_by_syntax(ra_fixture: &str) -> tt::Subtree { + let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap(); + let tt = syntax_node_to_token_tree(source_file.syntax()).unwrap().0; + + let parsed = parse_to_token_tree(ra_fixture).unwrap().0; + assert_eq!(tt, parsed); + + parsed +} + fn debug_dump_ignore_spaces(node: &ra_syntax::SyntaxNode) -> String { let mut level = 0; let mut buf = String::new(); -- cgit v1.2.3