diff options
Diffstat (limited to 'crates/ra_mbe/src/tt_cursor.rs')
-rw-r--r-- | crates/ra_mbe/src/tt_cursor.rs | 170 |
1 files changed, 168 insertions, 2 deletions
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs index 6f619621a..6ac3ac187 100644 --- a/crates/ra_mbe/src/tt_cursor.rs +++ b/crates/ra_mbe/src/tt_cursor.rs | |||
@@ -1,4 +1,124 @@ | |||
1 | use crate::ParseError; | 1 | use crate::ParseError; |
2 | use crate::syntax_bridge::{TtTokenSource, TtToken, TokenPeek}; | ||
3 | use ra_parser::{TokenSource, TreeSink}; | ||
4 | |||
5 | use ra_syntax::{ | ||
6 | SyntaxKind | ||
7 | }; | ||
8 | |||
9 | struct TtCursorTokenSource { | ||
10 | tt_pos: usize, | ||
11 | inner: TtTokenSource, | ||
12 | } | ||
13 | |||
14 | impl TtCursorTokenSource { | ||
15 | fn new(subtree: &tt::Subtree, curr: usize) -> TtCursorTokenSource { | ||
16 | let mut res = TtCursorTokenSource { inner: TtTokenSource::new(subtree), tt_pos: 1 }; | ||
17 | |||
18 | // Matching `TtToken` cursor to `tt::TokenTree` cursor | ||
19 | // It is because TtToken is not One to One mapping to tt::Token | ||
20 | // There are 3 case (`TtToken` <=> `tt::TokenTree`) : | ||
21 | // * One to One => ident, single char punch | ||
22 | // * Many to One => `tt::TokenTree::SubTree` | ||
23 | // * One to Many => multibyte punct | ||
24 | // | ||
25 | // Such that we cannot simpliy advance the cursor | ||
26 | // We have to bump it one by one | ||
27 | let mut pos = 0; | ||
28 | while pos < curr { | ||
29 | pos += res.bump(&subtree.token_trees[pos]); | ||
30 | } | ||
31 | |||
32 | res | ||
33 | } | ||
34 | |||
35 | fn skip_sibling_leaf(&self, leaf: &tt::Leaf, iter: &mut std::slice::Iter<tt::TokenTree>) { | ||
36 | if let tt::Leaf::Punct(p) = leaf { | ||
37 | let mut peek = TokenPeek::new(iter); | ||
38 | if let Some((_, _, _, size)) = TtTokenSource::convert_multi_char_punct(p, &mut peek) { | ||
39 | for _ in 0..size - 1 { | ||
40 | peek.next(); | ||
41 | } | ||
42 | } | ||
43 | } | ||
44 | } | ||
45 | |||
46 | fn count_tt_tokens( | ||
47 | &self, | ||
48 | tt: &tt::TokenTree, | ||
49 | iter: Option<&mut std::slice::Iter<tt::TokenTree>>, | ||
50 | ) -> usize { | ||
51 | assert!(!self.inner.tokens.is_empty()); | ||
52 | |||
53 | match tt { | ||
54 | tt::TokenTree::Subtree(sub_tree) => { | ||
55 | let mut iter = sub_tree.token_trees.iter(); | ||
56 | let mut count = match sub_tree.delimiter { | ||
57 | tt::Delimiter::None => 0, | ||
58 | _ => 2, | ||
59 | }; | ||
60 | |||
61 | while let Some(tt) = iter.next() { | ||
62 | count += self.count_tt_tokens(&tt, Some(&mut iter)); | ||
63 | } | ||
64 | count | ||
65 | } | ||
66 | |||
67 | tt::TokenTree::Leaf(leaf) => { | ||
68 | iter.map(|iter| { | ||
69 | self.skip_sibling_leaf(leaf, iter); | ||
70 | }); | ||
71 | |||
72 | 1 | ||
73 | } | ||
74 | } | ||
75 | } | ||
76 | |||
77 | fn count(&self, tt: &tt::TokenTree) -> usize { | ||
78 | self.count_tt_tokens(tt, None) | ||
79 | } | ||
80 | |||
81 | fn bump(&mut self, tt: &tt::TokenTree) -> usize { | ||
82 | let cur = self.current().unwrap(); | ||
83 | let n_tokens = cur.n_tokens; | ||
84 | self.tt_pos += self.count(tt); | ||
85 | n_tokens | ||
86 | } | ||
87 | |||
88 | fn current(&self) -> Option<&TtToken> { | ||
89 | self.inner.tokens.get(self.tt_pos) | ||
90 | } | ||
91 | } | ||
92 | |||
93 | impl TokenSource for TtCursorTokenSource { | ||
94 | fn token_kind(&self, pos: usize) -> SyntaxKind { | ||
95 | if let Some(tok) = self.inner.tokens.get(self.tt_pos + pos) { | ||
96 | tok.kind | ||
97 | } else { | ||
98 | SyntaxKind::EOF | ||
99 | } | ||
100 | } | ||
101 | fn is_token_joint_to_next(&self, pos: usize) -> bool { | ||
102 | self.inner.tokens[self.tt_pos + pos].is_joint_to_next | ||
103 | } | ||
104 | fn is_keyword(&self, pos: usize, kw: &str) -> bool { | ||
105 | self.inner.tokens[self.tt_pos + pos].text == *kw | ||
106 | } | ||
107 | } | ||
108 | |||
109 | struct TtCursorTokenSink { | ||
110 | token_pos: usize, | ||
111 | } | ||
112 | |||
113 | impl TreeSink for TtCursorTokenSink { | ||
114 | fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { | ||
115 | self.token_pos += n_tokens as usize; | ||
116 | } | ||
117 | |||
118 | fn start_node(&mut self, _kind: SyntaxKind) {} | ||
119 | fn finish_node(&mut self) {} | ||
120 | fn error(&mut self, _error: ra_parser::ParseError) {} | ||
121 | } | ||
2 | 122 | ||
3 | #[derive(Clone)] | 123 | #[derive(Clone)] |
4 | pub(crate) struct TtCursor<'a> { | 124 | pub(crate) struct TtCursor<'a> { |
@@ -78,8 +198,54 @@ impl<'a> TtCursor<'a> { | |||
78 | }) | 198 | }) |
79 | } | 199 | } |
80 | 200 | ||
81 | pub(crate) fn eat_path(&mut self) -> Option<tt::Subtree> { | 201 | fn eat_parse_result( |
82 | None | 202 | &mut self, |
203 | parsed_token: usize, | ||
204 | src: &mut TtCursorTokenSource, | ||
205 | ) -> Option<tt::TokenTree> { | ||
206 | let mut res = vec![]; | ||
207 | |||
208 | // Matching `TtToken` cursor to `tt::TokenTree` cursor | ||
209 | // It is because TtToken is not One to One mapping to tt::Token | ||
210 | // There are 3 case (`TtToken` <=> `tt::TokenTree`) : | ||
211 | // * One to One => ident, single char punch | ||
212 | // * Many to One => `tt::TokenTree::SubTree` | ||
213 | // * One to Many => multibyte punct | ||
214 | // | ||
215 | // Such that we cannot simpliy advance the cursor | ||
216 | // We have to bump it one by one | ||
217 | let next_pos = src.tt_pos + parsed_token; | ||
218 | while src.tt_pos < next_pos { | ||
219 | let n = src.bump(self.current().unwrap()); | ||
220 | res.extend((0..n).map(|_| self.eat().unwrap())); | ||
221 | } | ||
222 | |||
223 | let res: Vec<_> = res.into_iter().cloned().collect(); | ||
224 | |||
225 | match res.len() { | ||
226 | 0 => None, | ||
227 | 1 => Some(res[0].clone()), | ||
228 | _ => Some(tt::TokenTree::Subtree(tt::Subtree { | ||
229 | delimiter: tt::Delimiter::None, | ||
230 | token_trees: res, | ||
231 | })), | ||
232 | } | ||
233 | } | ||
234 | |||
235 | fn eat_parse<F>(&mut self, f: F) -> Option<tt::TokenTree> | ||
236 | where | ||
237 | F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), | ||
238 | { | ||
239 | let mut src = TtCursorTokenSource::new(self.subtree, self.pos); | ||
240 | let mut sink = TtCursorTokenSink { token_pos: 0 }; | ||
241 | |||
242 | f(&src, &mut sink); | ||
243 | |||
244 | self.eat_parse_result(sink.token_pos, &mut src) | ||
245 | } | ||
246 | |||
247 | pub(crate) fn eat_path(&mut self) -> Option<tt::TokenTree> { | ||
248 | self.eat_parse(ra_parser::parse_path) | ||
83 | } | 249 | } |
84 | 250 | ||
85 | pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { | 251 | pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { |