aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe/src/syntax_bridge.rs
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_mbe/src/syntax_bridge.rs')
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs29
1 files changed, 21 insertions, 8 deletions
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 521b96d68..3fe5abba3 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -129,17 +129,26 @@ impl TtTokenSource {
129 is_joint_to_next: false, 129 is_joint_to_next: false,
130 text: l.text.clone(), 130 text: l.text.clone(),
131 }, 131 },
132 tt::Leaf::Punct(p) => Tok { 132 tt::Leaf::Punct(p) => {
133 kind: SyntaxKind::from_char(p.char).unwrap(), 133 let kind = match p.char {
134 is_joint_to_next: p.spacing == tt::Spacing::Joint, 134 // lexer may produce combpund tokens for these ones
135 text: { 135 '.' => DOT,
136 ':' => COLON,
137 '=' => EQ,
138 '!' => EXCL,
139 '-' => MINUS,
140 c => SyntaxKind::from_char(c).unwrap(),
141 };
142 let text = {
136 let mut buf = [0u8; 4]; 143 let mut buf = [0u8; 4];
137 let s: &str = p.char.encode_utf8(&mut buf); 144 let s: &str = p.char.encode_utf8(&mut buf);
138 SmolStr::new(s) 145 SmolStr::new(s)
139 }, 146 };
140 }, 147 Tok { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
148 }
141 tt::Leaf::Ident(ident) => { 149 tt::Leaf::Ident(ident) => {
142 Tok { kind: IDENT, is_joint_to_next: false, text: ident.text.clone() } 150 let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT);
151 Tok { kind, is_joint_to_next: false, text: ident.text.clone() }
143 } 152 }
144 }; 153 };
145 self.tokens.push(tok) 154 self.tokens.push(tok)
@@ -161,7 +170,11 @@ impl TtTokenSource {
161 170
162impl TokenSource for TtTokenSource { 171impl TokenSource for TtTokenSource {
163 fn token_kind(&self, pos: usize) -> SyntaxKind { 172 fn token_kind(&self, pos: usize) -> SyntaxKind {
164 self.tokens[pos].kind 173 if let Some(tok) = self.tokens.get(pos) {
174 tok.kind
175 } else {
176 SyntaxKind::EOF
177 }
165 } 178 }
166 fn is_token_joint_to_next(&self, pos: usize) -> bool { 179 fn is_token_joint_to_next(&self, pos: usize) -> bool {
167 self.tokens[pos].is_joint_to_next 180 self.tokens[pos].is_joint_to_next