diff options
Diffstat (limited to 'crates/mbe/src/syntax_bridge.rs')
-rw-r--r-- | crates/mbe/src/syntax_bridge.rs | 63 |
1 files changed, 31 insertions, 32 deletions
diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 265c0d63d..e648519f9 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs | |||
@@ -70,15 +70,12 @@ pub fn token_tree_to_syntax_node( | |||
70 | tt: &tt::Subtree, | 70 | tt: &tt::Subtree, |
71 | fragment_kind: FragmentKind, | 71 | fragment_kind: FragmentKind, |
72 | ) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> { | 72 | ) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> { |
73 | let tmp; | 73 | let buffer = match tt { |
74 | let tokens = match tt { | 74 | tt::Subtree { delimiter: None, token_trees } => { |
75 | tt::Subtree { delimiter: None, token_trees } => token_trees.as_slice(), | 75 | TokenBuffer::from_tokens(token_trees.as_slice()) |
76 | _ => { | ||
77 | tmp = [tt.clone().into()]; | ||
78 | &tmp[..] | ||
79 | } | 76 | } |
77 | _ => TokenBuffer::from_subtree(tt), | ||
80 | }; | 78 | }; |
81 | let buffer = TokenBuffer::new(&tokens); | ||
82 | let mut token_source = SubtreeTokenSource::new(&buffer); | 79 | let mut token_source = SubtreeTokenSource::new(&buffer); |
83 | let mut tree_sink = TtTreeSink::new(buffer.begin()); | 80 | let mut tree_sink = TtTreeSink::new(buffer.begin()); |
84 | parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); | 81 | parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); |
@@ -152,7 +149,7 @@ impl TokenMap { | |||
152 | } | 149 | } |
153 | 150 | ||
154 | fn remove_delim(&mut self, idx: usize) { | 151 | fn remove_delim(&mut self, idx: usize) { |
155 | // FIXME: This could be accidently quadratic | 152 | // FIXME: This could be accidentally quadratic |
156 | self.entries.remove(idx); | 153 | self.entries.remove(idx); |
157 | } | 154 | } |
158 | } | 155 | } |
@@ -414,7 +411,7 @@ trait TokenConvertor { | |||
414 | fn id_alloc(&mut self) -> &mut TokenIdAlloc; | 411 | fn id_alloc(&mut self) -> &mut TokenIdAlloc; |
415 | } | 412 | } |
416 | 413 | ||
417 | impl<'a> SrcToken for (RawToken, &'a str) { | 414 | impl<'a> SrcToken for (&'a RawToken, &'a str) { |
418 | fn kind(&self) -> SyntaxKind { | 415 | fn kind(&self) -> SyntaxKind { |
419 | self.0.kind | 416 | self.0.kind |
420 | } | 417 | } |
@@ -431,7 +428,7 @@ impl<'a> SrcToken for (RawToken, &'a str) { | |||
431 | impl RawConvertor<'_> {} | 428 | impl RawConvertor<'_> {} |
432 | 429 | ||
433 | impl<'a> TokenConvertor for RawConvertor<'a> { | 430 | impl<'a> TokenConvertor for RawConvertor<'a> { |
434 | type Token = (RawToken, &'a str); | 431 | type Token = (&'a RawToken, &'a str); |
435 | 432 | ||
436 | fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> { | 433 | fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> { |
437 | convert_doc_comment(&doc_comment(token.1)) | 434 | convert_doc_comment(&doc_comment(token.1)) |
@@ -442,11 +439,11 @@ impl<'a> TokenConvertor for RawConvertor<'a> { | |||
442 | let range = TextRange::at(self.offset, token.len); | 439 | let range = TextRange::at(self.offset, token.len); |
443 | self.offset += token.len; | 440 | self.offset += token.len; |
444 | 441 | ||
445 | Some(((*token, &self.text[range]), range)) | 442 | Some(((token, &self.text[range]), range)) |
446 | } | 443 | } |
447 | 444 | ||
448 | fn peek(&self) -> Option<Self::Token> { | 445 | fn peek(&self) -> Option<Self::Token> { |
449 | let token = self.inner.as_slice().get(0).cloned(); | 446 | let token = self.inner.as_slice().get(0); |
450 | 447 | ||
451 | token.map(|it| { | 448 | token.map(|it| { |
452 | let range = TextRange::at(self.offset, it.len); | 449 | let range = TextRange::at(self.offset, it.len); |
@@ -479,14 +476,14 @@ impl Convertor { | |||
479 | 476 | ||
480 | #[derive(Debug)] | 477 | #[derive(Debug)] |
481 | enum SynToken { | 478 | enum SynToken { |
482 | Ordiniary(SyntaxToken), | 479 | Ordinary(SyntaxToken), |
483 | Punch(SyntaxToken, TextSize), | 480 | Punch(SyntaxToken, TextSize), |
484 | } | 481 | } |
485 | 482 | ||
486 | impl SynToken { | 483 | impl SynToken { |
487 | fn token(&self) -> &SyntaxToken { | 484 | fn token(&self) -> &SyntaxToken { |
488 | match self { | 485 | match self { |
489 | SynToken::Ordiniary(it) => it, | 486 | SynToken::Ordinary(it) => it, |
490 | SynToken::Punch(it, _) => it, | 487 | SynToken::Punch(it, _) => it, |
491 | } | 488 | } |
492 | } | 489 | } |
@@ -498,7 +495,7 @@ impl SrcToken for SynToken { | |||
498 | } | 495 | } |
499 | fn to_char(&self) -> Option<char> { | 496 | fn to_char(&self) -> Option<char> { |
500 | match self { | 497 | match self { |
501 | SynToken::Ordiniary(_) => None, | 498 | SynToken::Ordinary(_) => None, |
502 | SynToken::Punch(it, i) => it.text().chars().nth((*i).into()), | 499 | SynToken::Punch(it, i) => it.text().chars().nth((*i).into()), |
503 | } | 500 | } |
504 | } | 501 | } |
@@ -538,7 +535,7 @@ impl TokenConvertor for Convertor { | |||
538 | } else { | 535 | } else { |
539 | self.punct_offset = None; | 536 | self.punct_offset = None; |
540 | let range = curr.text_range(); | 537 | let range = curr.text_range(); |
541 | (SynToken::Ordiniary(curr), range) | 538 | (SynToken::Ordinary(curr), range) |
542 | }; | 539 | }; |
543 | 540 | ||
544 | Some(token) | 541 | Some(token) |
@@ -560,7 +557,7 @@ impl TokenConvertor for Convertor { | |||
560 | let token = if curr.kind().is_punct() { | 557 | let token = if curr.kind().is_punct() { |
561 | SynToken::Punch(curr, 0.into()) | 558 | SynToken::Punch(curr, 0.into()) |
562 | } else { | 559 | } else { |
563 | SynToken::Ordiniary(curr) | 560 | SynToken::Ordinary(curr) |
564 | }; | 561 | }; |
565 | Some(token) | 562 | Some(token) |
566 | } | 563 | } |
@@ -601,17 +598,16 @@ impl<'a> TtTreeSink<'a> { | |||
601 | } | 598 | } |
602 | } | 599 | } |
603 | 600 | ||
604 | fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> SmolStr { | 601 | fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> &'static str { |
605 | let texts = match d { | 602 | let texts = match d { |
606 | Some(tt::DelimiterKind::Parenthesis) => "()", | 603 | Some(tt::DelimiterKind::Parenthesis) => "()", |
607 | Some(tt::DelimiterKind::Brace) => "{}", | 604 | Some(tt::DelimiterKind::Brace) => "{}", |
608 | Some(tt::DelimiterKind::Bracket) => "[]", | 605 | Some(tt::DelimiterKind::Bracket) => "[]", |
609 | None => return "".into(), | 606 | None => return "", |
610 | }; | 607 | }; |
611 | 608 | ||
612 | let idx = closing as usize; | 609 | let idx = closing as usize; |
613 | let text = &texts[idx..texts.len() - (1 - idx)]; | 610 | &texts[idx..texts.len() - (1 - idx)] |
614 | text.into() | ||
615 | } | 611 | } |
616 | 612 | ||
617 | impl<'a> TreeSink for TtTreeSink<'a> { | 613 | impl<'a> TreeSink for TtTreeSink<'a> { |
@@ -626,29 +622,32 @@ impl<'a> TreeSink for TtTreeSink<'a> { | |||
626 | 622 | ||
627 | let mut last = self.cursor; | 623 | let mut last = self.cursor; |
628 | for _ in 0..n_tokens { | 624 | for _ in 0..n_tokens { |
625 | let tmp_str: SmolStr; | ||
629 | if self.cursor.eof() { | 626 | if self.cursor.eof() { |
630 | break; | 627 | break; |
631 | } | 628 | } |
632 | last = self.cursor; | 629 | last = self.cursor; |
633 | let text: SmolStr = match self.cursor.token_tree() { | 630 | let text: &str = match self.cursor.token_tree() { |
634 | Some(tt::TokenTree::Leaf(leaf)) => { | 631 | Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { |
635 | // Mark the range if needed | 632 | // Mark the range if needed |
636 | let (text, id) = match leaf { | 633 | let (text, id) = match leaf { |
637 | tt::Leaf::Ident(ident) => (ident.text.clone(), ident.id), | 634 | tt::Leaf::Ident(ident) => (&ident.text, ident.id), |
638 | tt::Leaf::Punct(punct) => { | 635 | tt::Leaf::Punct(punct) => { |
639 | assert!(punct.char.is_ascii()); | 636 | assert!(punct.char.is_ascii()); |
640 | let char = &(punct.char as u8); | 637 | let char = &(punct.char as u8); |
641 | let text = std::str::from_utf8(std::slice::from_ref(char)).unwrap(); | 638 | tmp_str = SmolStr::new_inline( |
642 | (SmolStr::new_inline(text), punct.id) | 639 | std::str::from_utf8(std::slice::from_ref(char)).unwrap(), |
640 | ); | ||
641 | (&tmp_str, punct.id) | ||
643 | } | 642 | } |
644 | tt::Leaf::Literal(lit) => (lit.text.clone(), lit.id), | 643 | tt::Leaf::Literal(lit) => (&lit.text, lit.id), |
645 | }; | 644 | }; |
646 | let range = TextRange::at(self.text_pos, TextSize::of(text.as_str())); | 645 | let range = TextRange::at(self.text_pos, TextSize::of(text.as_str())); |
647 | self.token_map.insert(id, range); | 646 | self.token_map.insert(id, range); |
648 | self.cursor = self.cursor.bump(); | 647 | self.cursor = self.cursor.bump(); |
649 | text | 648 | text |
650 | } | 649 | } |
651 | Some(tt::TokenTree::Subtree(subtree)) => { | 650 | Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { |
652 | self.cursor = self.cursor.subtree().unwrap(); | 651 | self.cursor = self.cursor.subtree().unwrap(); |
653 | if let Some(id) = subtree.delimiter.map(|it| it.id) { | 652 | if let Some(id) = subtree.delimiter.map(|it| it.id) { |
654 | self.open_delims.insert(id, self.text_pos); | 653 | self.open_delims.insert(id, self.text_pos); |
@@ -672,7 +671,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { | |||
672 | } | 671 | } |
673 | }; | 672 | }; |
674 | self.buf += &text; | 673 | self.buf += &text; |
675 | self.text_pos += TextSize::of(text.as_str()); | 674 | self.text_pos += TextSize::of(text); |
676 | } | 675 | } |
677 | 676 | ||
678 | let text = SmolStr::new(self.buf.as_str()); | 677 | let text = SmolStr::new(self.buf.as_str()); |
@@ -682,8 +681,8 @@ impl<'a> TreeSink for TtTreeSink<'a> { | |||
682 | // Add whitespace between adjoint puncts | 681 | // Add whitespace between adjoint puncts |
683 | let next = last.bump(); | 682 | let next = last.bump(); |
684 | if let ( | 683 | if let ( |
685 | Some(tt::TokenTree::Leaf(tt::Leaf::Punct(curr))), | 684 | Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(curr), _)), |
686 | Some(tt::TokenTree::Leaf(tt::Leaf::Punct(_))), | 685 | Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(_), _)), |
687 | ) = (last.token_tree(), next.token_tree()) | 686 | ) = (last.token_tree(), next.token_tree()) |
688 | { | 687 | { |
689 | // Note: We always assume the semi-colon would be the last token in | 688 | // Note: We always assume the semi-colon would be the last token in |
@@ -742,7 +741,7 @@ mod tests { | |||
742 | ) | 741 | ) |
743 | .expand_tt("literals!(foo);"); | 742 | .expand_tt("literals!(foo);"); |
744 | let tts = &[expansion.into()]; | 743 | let tts = &[expansion.into()]; |
745 | let buffer = tt::buffer::TokenBuffer::new(tts); | 744 | let buffer = tt::buffer::TokenBuffer::from_tokens(tts); |
746 | let mut tt_src = SubtreeTokenSource::new(&buffer); | 745 | let mut tt_src = SubtreeTokenSource::new(&buffer); |
747 | let mut tokens = vec![]; | 746 | let mut tokens = vec![]; |
748 | while tt_src.current().kind != EOF { | 747 | while tt_src.current().kind != EOF { |