diff options
author | Edwin Cheng <[email protected]> | 2019-04-06 13:14:28 +0100 |
---|---|---|
committer | Edwin Cheng <[email protected]> | 2019-04-06 13:14:28 +0100 |
commit | aac9dfa46418603940ab2333cfea2190d9464d9e (patch) | |
tree | 83bc019e9961703ba1025c647a92ba4cd8e394af /crates/ra_mbe/src/syntax_bridge.rs | |
parent | 1d7735fbc6795c3ea5f02950b47413e0b35d6677 (diff) |
Add TtCursorTokenSource and TtCursorTokenSink
Diffstat (limited to 'crates/ra_mbe/src/syntax_bridge.rs')
-rw-r--r-- | crates/ra_mbe/src/syntax_bridge.rs | 58 |
1 files changed, 36 insertions, 22 deletions
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 139a0fd33..3a0702a30 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs | |||
@@ -104,15 +104,16 @@ fn convert_tt( | |||
104 | } | 104 | } |
105 | 105 | ||
106 | #[derive(Debug)] | 106 | #[derive(Debug)] |
107 | struct TtTokenSource { | 107 | pub(crate) struct TtTokenSource { |
108 | tokens: Vec<TtToken>, | 108 | pub tokens: Vec<TtToken>, |
109 | } | 109 | } |
110 | 110 | ||
111 | #[derive(Debug)] | 111 | #[derive(Debug)] |
112 | struct TtToken { | 112 | pub(crate) struct TtToken { |
113 | kind: SyntaxKind, | 113 | pub kind: SyntaxKind, |
114 | is_joint_to_next: bool, | 114 | pub is_joint_to_next: bool, |
115 | text: SmolStr, | 115 | pub text: SmolStr, |
116 | pub n_tokens: usize, | ||
116 | } | 117 | } |
117 | 118 | ||
118 | // Some helper functions | 119 | // Some helper functions |
@@ -123,7 +124,7 @@ fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> { | |||
123 | None | 124 | None |
124 | } | 125 | } |
125 | 126 | ||
126 | struct TokenPeek<'a, I> | 127 | pub(crate) struct TokenPeek<'a, I> |
127 | where | 128 | where |
128 | I: Iterator<Item = &'a tt::TokenTree>, | 129 | I: Iterator<Item = &'a tt::TokenTree>, |
129 | { | 130 | { |
@@ -134,7 +135,11 @@ impl<'a, I> TokenPeek<'a, I> | |||
134 | where | 135 | where |
135 | I: Iterator<Item = &'a tt::TokenTree>, | 136 | I: Iterator<Item = &'a tt::TokenTree>, |
136 | { | 137 | { |
137 | fn next(&mut self) -> Option<&tt::TokenTree> { | 138 | pub fn new(iter: I) -> Self { |
139 | TokenPeek { iter: itertools::multipeek(iter) } | ||
140 | } | ||
141 | |||
142 | pub fn next(&mut self) -> Option<&tt::TokenTree> { | ||
138 | self.iter.next() | 143 | self.iter.next() |
139 | } | 144 | } |
140 | 145 | ||
@@ -161,14 +166,14 @@ where | |||
161 | } | 166 | } |
162 | 167 | ||
163 | impl TtTokenSource { | 168 | impl TtTokenSource { |
164 | fn new(tt: &tt::Subtree) -> TtTokenSource { | 169 | pub fn new(tt: &tt::Subtree) -> TtTokenSource { |
165 | let mut res = TtTokenSource { tokens: Vec::new() }; | 170 | let mut res = TtTokenSource { tokens: Vec::new() }; |
166 | res.convert_subtree(tt); | 171 | res.convert_subtree(tt); |
167 | res | 172 | res |
168 | } | 173 | } |
169 | fn convert_subtree(&mut self, sub: &tt::Subtree) { | 174 | fn convert_subtree(&mut self, sub: &tt::Subtree) { |
170 | self.push_delim(sub.delimiter, false); | 175 | self.push_delim(sub.delimiter, false); |
171 | let mut peek = TokenPeek { iter: itertools::multipeek(sub.token_trees.iter()) }; | 176 | let mut peek = TokenPeek::new(sub.token_trees.iter()); |
172 | while let Some(tt) = peek.iter.next() { | 177 | while let Some(tt) = peek.iter.next() { |
173 | self.convert_tt(tt, &mut peek); | 178 | self.convert_tt(tt, &mut peek); |
174 | } | 179 | } |
@@ -194,10 +199,17 @@ impl TtTokenSource { | |||
194 | kind: classify_literal(&l.text).unwrap().kind, | 199 | kind: classify_literal(&l.text).unwrap().kind, |
195 | is_joint_to_next: false, | 200 | is_joint_to_next: false, |
196 | text: l.text.clone(), | 201 | text: l.text.clone(), |
202 | n_tokens: 1, | ||
197 | }, | 203 | }, |
198 | tt::Leaf::Punct(p) => { | 204 | tt::Leaf::Punct(p) => { |
199 | if let Some(tt) = Self::convert_multi_char_punct(p, iter) { | 205 | if let Some((kind, is_joint_to_next, text, size)) = |
200 | tt | 206 | Self::convert_multi_char_punct(p, iter) |
207 | { | ||
208 | for _ in 0..size - 1 { | ||
209 | iter.next(); | ||
210 | } | ||
211 | |||
212 | TtToken { kind, is_joint_to_next, text: text.into(), n_tokens: size } | ||
201 | } else { | 213 | } else { |
202 | let kind = match p.char { | 214 | let kind = match p.char { |
203 | // lexer may produce combpund tokens for these ones | 215 | // lexer may produce combpund tokens for these ones |
@@ -213,21 +225,26 @@ impl TtTokenSource { | |||
213 | let s: &str = p.char.encode_utf8(&mut buf); | 225 | let s: &str = p.char.encode_utf8(&mut buf); |
214 | SmolStr::new(s) | 226 | SmolStr::new(s) |
215 | }; | 227 | }; |
216 | TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text } | 228 | TtToken { |
229 | kind, | ||
230 | is_joint_to_next: p.spacing == tt::Spacing::Joint, | ||
231 | text, | ||
232 | n_tokens: 1, | ||
233 | } | ||
217 | } | 234 | } |
218 | } | 235 | } |
219 | tt::Leaf::Ident(ident) => { | 236 | tt::Leaf::Ident(ident) => { |
220 | let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT); | 237 | let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT); |
221 | TtToken { kind, is_joint_to_next: false, text: ident.text.clone() } | 238 | TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } |
222 | } | 239 | } |
223 | }; | 240 | }; |
224 | self.tokens.push(tok) | 241 | self.tokens.push(tok) |
225 | } | 242 | } |
226 | 243 | ||
227 | fn convert_multi_char_punct<'a, I>( | 244 | pub(crate) fn convert_multi_char_punct<'a, I>( |
228 | p: &tt::Punct, | 245 | p: &tt::Punct, |
229 | iter: &mut TokenPeek<'a, I>, | 246 | iter: &mut TokenPeek<'a, I>, |
230 | ) -> Option<TtToken> | 247 | ) -> Option<(SyntaxKind, bool, &'static str, usize)> |
231 | where | 248 | where |
232 | I: Iterator<Item = &'a tt::TokenTree>, | 249 | I: Iterator<Item = &'a tt::TokenTree>, |
233 | { | 250 | { |
@@ -239,9 +256,7 @@ impl TtTokenSource { | |||
239 | ('.', '.', '=') => Some((DOTDOTEQ, "..=")), | 256 | ('.', '.', '=') => Some((DOTDOTEQ, "..=")), |
240 | _ => None, | 257 | _ => None, |
241 | } { | 258 | } { |
242 | iter.next(); | 259 | return Some((kind, is_joint_to_next, text, 3)); |
243 | iter.next(); | ||
244 | return Some(TtToken { kind, is_joint_to_next, text: text.into() }); | ||
245 | } | 260 | } |
246 | } | 261 | } |
247 | 262 | ||
@@ -273,8 +288,7 @@ impl TtTokenSource { | |||
273 | 288 | ||
274 | _ => None, | 289 | _ => None, |
275 | } { | 290 | } { |
276 | iter.next(); | 291 | return Some((kind, is_joint_to_next, text, 2)); |
277 | return Some(TtToken { kind, is_joint_to_next, text: text.into() }); | ||
278 | } | 292 | } |
279 | } | 293 | } |
280 | 294 | ||
@@ -291,7 +305,7 @@ impl TtTokenSource { | |||
291 | let idx = closing as usize; | 305 | let idx = closing as usize; |
292 | let kind = kinds[idx]; | 306 | let kind = kinds[idx]; |
293 | let text = &texts[idx..texts.len() - (1 - idx)]; | 307 | let text = &texts[idx..texts.len() - (1 - idx)]; |
294 | let tok = TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) }; | 308 | let tok = TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 }; |
295 | self.tokens.push(tok) | 309 | self.tokens.push(tok) |
296 | } | 310 | } |
297 | } | 311 | } |