aboutsummaryrefslogtreecommitdiff
path: root/crates/mbe/src/subtree_source.rs
diff options
context:
space:
mode:
Diffstat (limited to 'crates/mbe/src/subtree_source.rs')
-rw-r--r--crates/mbe/src/subtree_source.rs17
1 files changed, 12 insertions, 5 deletions
diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs
index 41461b315..8941da0f1 100644
--- a/crates/mbe/src/subtree_source.rs
+++ b/crates/mbe/src/subtree_source.rs
@@ -2,7 +2,7 @@
2 2
3use parser::{Token, TokenSource}; 3use parser::{Token, TokenSource};
4use std::cell::{Cell, Ref, RefCell}; 4use std::cell::{Cell, Ref, RefCell};
5use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; 5use syntax::{tokenize, SmolStr, SyntaxKind, SyntaxKind::*, T};
6use tt::buffer::{Cursor, TokenBuffer}; 6use tt::buffer::{Cursor, TokenBuffer};
7 7
8#[derive(Debug, Clone, Eq, PartialEq)] 8#[derive(Debug, Clone, Eq, PartialEq)]
@@ -155,10 +155,17 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
155} 155}
156 156
157fn convert_literal(l: &tt::Literal) -> TtToken { 157fn convert_literal(l: &tt::Literal) -> TtToken {
158 let kind = lex_single_syntax_kind(&l.text) 158 let mut kinds = tokenize(&l.text).0.into_iter().map(|token| token.kind);
159 .map(|(kind, _error)| kind) 159
160 .filter(|kind| kind.is_literal()) 160 let kind = match kinds.next() {
161 .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); 161 Some(kind) if kind.is_literal() => Some(kind),
162 Some(SyntaxKind::MINUS) => match kinds.next() {
163 Some(kind) if kind.is_literal() => Some(kind),
164 _ => None,
165 },
166 _ => None,
167 }
168 .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
162 169
163 TtToken { kind, is_joint_to_next: false, text: l.text.clone() } 170 TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
164} 171}