diff options
Diffstat (limited to 'crates/ra_mbe/src/syntax_bridge.rs')
-rw-r--r-- | crates/ra_mbe/src/syntax_bridge.rs | 29 |
1 files changed, 21 insertions, 8 deletions
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 521b96d68..3fe5abba3 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs | |||
@@ -129,17 +129,26 @@ impl TtTokenSource { | |||
129 | is_joint_to_next: false, | 129 | is_joint_to_next: false, |
130 | text: l.text.clone(), | 130 | text: l.text.clone(), |
131 | }, | 131 | }, |
132 | tt::Leaf::Punct(p) => Tok { | 132 | tt::Leaf::Punct(p) => { |
133 | kind: SyntaxKind::from_char(p.char).unwrap(), | 133 | let kind = match p.char { |
134 | is_joint_to_next: p.spacing == tt::Spacing::Joint, | 134 | // lexer may produce combpund tokens for these ones |
135 | text: { | 135 | '.' => DOT, |
136 | ':' => COLON, | ||
137 | '=' => EQ, | ||
138 | '!' => EXCL, | ||
139 | '-' => MINUS, | ||
140 | c => SyntaxKind::from_char(c).unwrap(), | ||
141 | }; | ||
142 | let text = { | ||
136 | let mut buf = [0u8; 4]; | 143 | let mut buf = [0u8; 4]; |
137 | let s: &str = p.char.encode_utf8(&mut buf); | 144 | let s: &str = p.char.encode_utf8(&mut buf); |
138 | SmolStr::new(s) | 145 | SmolStr::new(s) |
139 | }, | 146 | }; |
140 | }, | 147 | Tok { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text } |
148 | } | ||
141 | tt::Leaf::Ident(ident) => { | 149 | tt::Leaf::Ident(ident) => { |
142 | Tok { kind: IDENT, is_joint_to_next: false, text: ident.text.clone() } | 150 | let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT); |
151 | Tok { kind, is_joint_to_next: false, text: ident.text.clone() } | ||
143 | } | 152 | } |
144 | }; | 153 | }; |
145 | self.tokens.push(tok) | 154 | self.tokens.push(tok) |
@@ -161,7 +170,11 @@ impl TtTokenSource { | |||
161 | 170 | ||
162 | impl TokenSource for TtTokenSource { | 171 | impl TokenSource for TtTokenSource { |
163 | fn token_kind(&self, pos: usize) -> SyntaxKind { | 172 | fn token_kind(&self, pos: usize) -> SyntaxKind { |
164 | self.tokens[pos].kind | 173 | if let Some(tok) = self.tokens.get(pos) { |
174 | tok.kind | ||
175 | } else { | ||
176 | SyntaxKind::EOF | ||
177 | } | ||
165 | } | 178 | } |
166 | fn is_token_joint_to_next(&self, pos: usize) -> bool { | 179 | fn is_token_joint_to_next(&self, pos: usize) -> bool { |
167 | self.tokens[pos].is_joint_to_next | 180 | self.tokens[pos].is_joint_to_next |