aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorEdwin Cheng <[email protected]>2020-03-19 12:40:25 +0000
committerEdwin Cheng <[email protected]>2020-03-20 15:36:29 +0000
commit27c516970bea6bee9d6bca52ac9ad619412809ef (patch)
treea28fa39e9c72861524f9d6f263f960d9aab83272
parent92b561b5c7da8303473792ba2bacb430614da2d1 (diff)
Add test for delim bug
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs103
-rw-r--r--crates/ra_mbe/src/tests.rs56
2 files changed, 147 insertions, 12 deletions
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index fcb73fbc7..0678c37ee 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -267,6 +267,108 @@ struct RawConvertor<'a> {
267 inner: std::slice::Iter<'a, Token>, 267 inner: std::slice::Iter<'a, Token>,
268} 268}
269 269
270trait SrcToken {
271 fn kind() -> SyntaxKind;
272}
273
274trait TokenConvertor {
275 type Token : SrcToken;
276
277 fn go(&mut self) -> Option<tt::Subtree> {
278 let mut subtree = tt::Subtree::default();
279 subtree.delimiter = None;
280 while self.peek().is_some() {
281 self.collect_leaf(&mut subtree.token_trees);
282 }
283 if subtree.token_trees.is_empty() {
284 return None;
285 }
286 if subtree.token_trees.len() == 1 {
287 if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
288 return Some(first.clone());
289 }
290 }
291 Some(subtree)
292 }
293
294 fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
295
296 fn peek(&self) -> Option<Self::Token>;
297
298 fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
299 let (token, range) = match self.bump() {
300 None => return,
301 Some(it) => it,
302 };
303
304 let k: SyntaxKind = token.kind();
305 if k == COMMENT {
306 let node = doc_comment(&self.text[range]);
307 if let Some(tokens) = convert_doc_comment(&node) {
308 result.extend(tokens);
309 }
310 return;
311 }
312
313 result.push(if k.is_punct() {
314 let delim = match k {
315 T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
316 T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])),
317 T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])),
318 _ => None,
319 };
320
321 if let Some((kind, closed)) = delim {
322 let mut subtree = tt::Subtree::default();
323 let id = self.id_alloc.open_delim(range);
324 subtree.delimiter = Some(tt::Delimiter { kind, id });
325
326 while self.peek().map(|it| it.kind != closed).unwrap_or(false) {
327 self.collect_leaf(&mut subtree.token_trees);
328 }
329 let last_range = match self.bump() {
330 None => return,
331 Some(it) => it.1,
332 };
333 self.id_alloc.close_delim(id, last_range);
334 subtree.into()
335 } else {
336 let spacing = match self.peek() {
337 Some(next)
338 if next.kind.is_trivia()
339 || next.kind == T!['[']
340 || next.kind == T!['{']
341 || next.kind == T!['('] =>
342 {
343 tt::Spacing::Alone
344 }
345 Some(next) if next.kind.is_punct() => tt::Spacing::Joint,
346 _ => tt::Spacing::Alone,
347 };
348 let char =
349 self.text[range].chars().next().expect("Token from lexer must be single char");
350
351 tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc.alloc(range) }).into()
352 }
353 } else {
354 macro_rules! make_leaf {
355 ($i:ident) => {
356 tt::$i { id: self.id_alloc.alloc(range), text: self.text[range].into() }.into()
357 };
358 }
359 let leaf: tt::Leaf = match k {
360 T![true] | T![false] => make_leaf!(Literal),
361 IDENT | LIFETIME => make_leaf!(Ident),
362 k if k.is_keyword() => make_leaf!(Ident),
363 k if k.is_literal() => make_leaf!(Literal),
364 _ => return,
365 };
366
367 leaf.into()
368 });
369 }
370}
371
270impl RawConvertor<'_> { 372impl RawConvertor<'_> {
271 fn go(&mut self) -> Option<tt::Subtree> { 373 fn go(&mut self) -> Option<tt::Subtree> {
272 let mut subtree = tt::Subtree::default(); 374 let mut subtree = tt::Subtree::default();
@@ -295,6 +397,7 @@ impl RawConvertor<'_> {
295 fn peek(&self) -> Option<Token> { 397 fn peek(&self) -> Option<Token> {
296 self.inner.as_slice().get(0).cloned() 398 self.inner.as_slice().get(0).cloned()
297 } 399 }
400
298 401
299 fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) { 402 fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
300 let (token, range) = match self.bump() { 403 let (token, range) = match self.bump() {
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 44f381938..966af1d12 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -427,22 +427,28 @@ MACRO_ITEMS@[0; 40)
427 ); 427 );
428} 428}
429 429
430#[test] 430fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree {
431fn test_expand_literals_to_token_tree() { 431 if let tt::TokenTree::Subtree(subtree) = tt {
432 fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { 432 return &subtree;
433 if let tt::TokenTree::Subtree(subtree) = tt {
434 return &subtree;
435 }
436 unreachable!("It is not a subtree");
437 } 433 }
434 unreachable!("It is not a subtree");
435}
436fn to_literal(tt: &tt::TokenTree) -> &tt::Literal {
437 if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt {
438 return lit;
439 }
440 unreachable!("It is not a literal");
441}
438 442
439 fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { 443fn to_punct(tt: &tt::TokenTree) -> &tt::Punct {
440 if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { 444 if let tt::TokenTree::Leaf(tt::Leaf::Punct(lit)) = tt {
441 return lit; 445 return lit;
442 }
443 unreachable!("It is not a literal");
444 } 446 }
447 unreachable!("It is not a Punct");
448}
445 449
450#[test]
451fn test_expand_literals_to_token_tree() {
446 let expansion = parse_macro( 452 let expansion = parse_macro(
447 r#" 453 r#"
448 macro_rules! literals { 454 macro_rules! literals {
@@ -471,6 +477,22 @@ fn test_expand_literals_to_token_tree() {
471} 477}
472 478
473#[test] 479#[test]
480fn test_attr_to_token_tree() {
481 let expansion = parse_to_token_tree_by_syntax(
482 r#"
483 #[derive(Copy)]
484 struct Foo;
485 "#,
486 );
487
488 assert_eq!(to_punct(&expansion.token_trees[0]).char, '#');
489 assert_eq!(
490 to_subtree(&expansion.token_trees[1]).delimiter_kind(),
491 Some(tt::DelimiterKind::Bracket)
492 );
493}
494
495#[test]
474fn test_two_idents() { 496fn test_two_idents() {
475 parse_macro( 497 parse_macro(
476 r#" 498 r#"
@@ -1517,6 +1539,16 @@ pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture {
1517 MacroFixture { rules } 1539 MacroFixture { rules }
1518} 1540}
1519 1541
1542pub(crate) fn parse_to_token_tree_by_syntax(ra_fixture: &str) -> tt::Subtree {
1543 let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
1544 let tt = syntax_node_to_token_tree(source_file.syntax()).unwrap().0;
1545
1546 let parsed = parse_to_token_tree(ra_fixture).unwrap().0;
1547 assert_eq!(tt, parsed);
1548
1549 parsed
1550}
1551
1520fn debug_dump_ignore_spaces(node: &ra_syntax::SyntaxNode) -> String { 1552fn debug_dump_ignore_spaces(node: &ra_syntax::SyntaxNode) -> String {
1521 let mut level = 0; 1553 let mut level = 0;
1522 let mut buf = String::new(); 1554 let mut buf = String::new();