diff options
Diffstat (limited to 'crates/ra_mbe/src/subtree_source.rs')
-rw-r--r-- | crates/ra_mbe/src/subtree_source.rs | 262 |
1 files changed, 70 insertions, 192 deletions
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index c938acf64..972af4a7c 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs | |||
@@ -1,50 +1,10 @@ | |||
1 | use ra_parser::{TokenSource}; | 1 | use ra_parser::{TokenSource}; |
2 | use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T}; | 2 | use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T}; |
3 | use std::cell::{RefCell}; | 3 | use std::cell::{RefCell, Cell}; |
4 | use tt::buffer::{TokenBuffer, Cursor}; | ||
4 | 5 | ||
5 | // A Sequece of Token, | 6 | pub(crate) trait Querier { |
6 | #[derive(Debug, Clone, Eq, PartialEq)] | 7 | fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool); |
7 | pub(super) enum TokenSeq<'a> { | ||
8 | Subtree(&'a tt::Subtree), | ||
9 | Seq(&'a [tt::TokenTree]), | ||
10 | } | ||
11 | |||
12 | impl<'a> From<&'a tt::Subtree> for TokenSeq<'a> { | ||
13 | fn from(s: &'a tt::Subtree) -> TokenSeq<'a> { | ||
14 | TokenSeq::Subtree(s) | ||
15 | } | ||
16 | } | ||
17 | |||
18 | impl<'a> From<&'a [tt::TokenTree]> for TokenSeq<'a> { | ||
19 | fn from(s: &'a [tt::TokenTree]) -> TokenSeq<'a> { | ||
20 | TokenSeq::Seq(s) | ||
21 | } | ||
22 | } | ||
23 | |||
24 | #[derive(Debug)] | ||
25 | enum DelimToken<'a> { | ||
26 | Delim(&'a tt::Delimiter, bool), | ||
27 | Token(&'a tt::TokenTree), | ||
28 | End, | ||
29 | } | ||
30 | |||
31 | impl<'a> TokenSeq<'a> { | ||
32 | fn get(&self, pos: usize) -> DelimToken<'a> { | ||
33 | match self { | ||
34 | TokenSeq::Subtree(subtree) => { | ||
35 | let len = subtree.token_trees.len() + 2; | ||
36 | match pos { | ||
37 | p if p >= len => DelimToken::End, | ||
38 | p if p == len - 1 => DelimToken::Delim(&subtree.delimiter, true), | ||
39 | 0 => DelimToken::Delim(&subtree.delimiter, false), | ||
40 | p => DelimToken::Token(&subtree.token_trees[p - 1]), | ||
41 | } | ||
42 | } | ||
43 | TokenSeq::Seq(tokens) => { | ||
44 | tokens.get(pos).map(DelimToken::Token).unwrap_or(DelimToken::End) | ||
45 | } | ||
46 | } | ||
47 | } | ||
48 | } | 8 | } |
49 | 9 | ||
50 | #[derive(Debug, Clone, Eq, PartialEq)] | 10 | #[derive(Debug, Clone, Eq, PartialEq)] |
@@ -54,183 +14,101 @@ struct TtToken { | |||
54 | pub text: SmolStr, | 14 | pub text: SmolStr, |
55 | } | 15 | } |
56 | 16 | ||
57 | #[derive(Debug, Clone, Eq, PartialEq)] | ||
58 | enum WalkCursor { | ||
59 | Token(usize, TtToken), | ||
60 | Eof, | ||
61 | } | ||
62 | |||
63 | #[derive(Debug)] | ||
64 | struct SubTreeWalker<'a> { | ||
65 | pos: usize, | ||
66 | stack: Vec<(TokenSeq<'a>, usize)>, | ||
67 | cursor: WalkCursor, | ||
68 | ts: TokenSeq<'a>, | ||
69 | } | ||
70 | |||
71 | impl<'a> SubTreeWalker<'a> { | ||
72 | fn new(ts: TokenSeq<'a>) -> SubTreeWalker { | ||
73 | let mut res = SubTreeWalker { pos: 0, stack: vec![], cursor: WalkCursor::Eof, ts }; | ||
74 | |||
75 | res.reset(); | ||
76 | res | ||
77 | } | ||
78 | |||
79 | fn is_eof(&self) -> bool { | ||
80 | self.cursor == WalkCursor::Eof | ||
81 | } | ||
82 | |||
83 | fn reset(&mut self) { | ||
84 | self.pos = 0; | ||
85 | self.stack = vec![]; | ||
86 | |||
87 | self.cursor = match self.ts.get(0) { | ||
88 | DelimToken::Token(token) => match token { | ||
89 | tt::TokenTree::Subtree(subtree) => { | ||
90 | let ts = TokenSeq::from(subtree); | ||
91 | self.stack.push((ts, 0)); | ||
92 | WalkCursor::Token(0, convert_delim(subtree.delimiter, false)) | ||
93 | } | ||
94 | tt::TokenTree::Leaf(leaf) => WalkCursor::Token(0, convert_leaf(leaf)), | ||
95 | }, | ||
96 | DelimToken::Delim(delim, is_end) => { | ||
97 | assert!(!is_end); | ||
98 | WalkCursor::Token(0, convert_delim(*delim, false)) | ||
99 | } | ||
100 | DelimToken::End => WalkCursor::Eof, | ||
101 | } | ||
102 | } | ||
103 | |||
104 | fn current(&self) -> Option<&TtToken> { | ||
105 | match &self.cursor { | ||
106 | WalkCursor::Token(_, t) => Some(t), | ||
107 | WalkCursor::Eof => None, | ||
108 | } | ||
109 | } | ||
110 | |||
111 | fn top(&self) -> &TokenSeq { | ||
112 | self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts) | ||
113 | } | ||
114 | |||
115 | /// Move cursor forward by 1 step | ||
116 | fn forward(&mut self) { | ||
117 | if self.is_eof() { | ||
118 | return; | ||
119 | } | ||
120 | self.pos += 1; | ||
121 | |||
122 | if let WalkCursor::Token(u, _) = self.cursor { | ||
123 | self.cursor = self.walk_token(u) | ||
124 | } | ||
125 | } | ||
126 | |||
127 | /// Traversal child token | ||
128 | fn walk_token(&mut self, pos: usize) -> WalkCursor { | ||
129 | let top = self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts); | ||
130 | let pos = pos + 1; | ||
131 | |||
132 | match top.get(pos) { | ||
133 | DelimToken::Token(token) => match token { | ||
134 | tt::TokenTree::Subtree(subtree) => { | ||
135 | let ts = TokenSeq::from(subtree); | ||
136 | self.stack.push((ts, pos)); | ||
137 | WalkCursor::Token(0, convert_delim(subtree.delimiter, false)) | ||
138 | } | ||
139 | tt::TokenTree::Leaf(leaf) => WalkCursor::Token(pos, convert_leaf(leaf)), | ||
140 | }, | ||
141 | DelimToken::Delim(delim, is_end) => { | ||
142 | WalkCursor::Token(pos, convert_delim(*delim, is_end)) | ||
143 | } | ||
144 | DelimToken::End => { | ||
145 | // it is the top level | ||
146 | if let Some((_, last_idx)) = self.stack.pop() { | ||
147 | self.walk_token(last_idx) | ||
148 | } else { | ||
149 | WalkCursor::Eof | ||
150 | } | ||
151 | } | ||
152 | } | ||
153 | } | ||
154 | } | ||
155 | |||
156 | pub(crate) trait Querier { | ||
157 | fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool); | ||
158 | } | ||
159 | |||
160 | // A wrapper class for ref cell | 17 | // A wrapper class for ref cell |
161 | #[derive(Debug)] | 18 | #[derive(Debug)] |
162 | pub(crate) struct WalkerOwner<'a> { | 19 | pub(crate) struct SubtreeWalk<'a> { |
163 | walker: RefCell<SubTreeWalker<'a>>, | 20 | start: Cursor<'a>, |
21 | cursor: Cell<Cursor<'a>>, | ||
164 | cached: RefCell<Vec<Option<TtToken>>>, | 22 | cached: RefCell<Vec<Option<TtToken>>>, |
165 | } | 23 | } |
166 | 24 | ||
167 | impl<'a> WalkerOwner<'a> { | 25 | impl<'a> SubtreeWalk<'a> { |
168 | fn new<I: Into<TokenSeq<'a>>>(ts: I) -> Self { | 26 | fn new(cursor: Cursor<'a>) -> Self { |
169 | WalkerOwner { | 27 | SubtreeWalk { |
170 | walker: RefCell::new(SubTreeWalker::new(ts.into())), | 28 | start: cursor, |
29 | cursor: Cell::new(cursor), | ||
171 | cached: RefCell::new(Vec::with_capacity(10)), | 30 | cached: RefCell::new(Vec::with_capacity(10)), |
172 | } | 31 | } |
173 | } | 32 | } |
174 | 33 | ||
175 | fn get<'b>(&self, pos: usize) -> Option<TtToken> { | 34 | fn get(&self, pos: usize) -> Option<TtToken> { |
176 | let mut cached = self.cached.borrow_mut(); | 35 | let mut cached = self.cached.borrow_mut(); |
177 | if pos < cached.len() { | 36 | if pos < cached.len() { |
178 | return cached[pos].clone(); | 37 | return cached[pos].clone(); |
179 | } | 38 | } |
180 | 39 | ||
181 | while pos >= cached.len() { | 40 | while pos >= cached.len() { |
182 | self.set_pos(cached.len()); | 41 | let cursor = self.cursor.get(); |
183 | let walker = self.walker.borrow(); | 42 | if cursor.eof() { |
184 | cached.push(walker.current().cloned()); | 43 | cached.push(None); |
44 | continue; | ||
45 | } | ||
46 | |||
47 | match cursor.token_tree() { | ||
48 | Some(tt::TokenTree::Leaf(leaf)) => { | ||
49 | cached.push(Some(convert_leaf(&leaf))); | ||
50 | self.cursor.set(cursor.bump()); | ||
51 | } | ||
52 | Some(tt::TokenTree::Subtree(subtree)) => { | ||
53 | self.cursor.set(cursor.subtree().unwrap()); | ||
54 | cached.push(Some(convert_delim(subtree.delimiter, false))); | ||
55 | } | ||
56 | None => { | ||
57 | if let Some(subtree) = cursor.end() { | ||
58 | cached.push(Some(convert_delim(subtree.delimiter, true))); | ||
59 | self.cursor.set(cursor.bump()); | ||
60 | } | ||
61 | } | ||
62 | } | ||
185 | } | 63 | } |
186 | 64 | ||
187 | return cached[pos].clone(); | 65 | return cached[pos].clone(); |
188 | } | 66 | } |
189 | 67 | ||
190 | fn set_pos(&self, pos: usize) { | 68 | fn collect_token_trees(&mut self, n: usize) -> Vec<tt::TokenTree> { |
191 | let mut walker = self.walker.borrow_mut(); | 69 | let mut res = vec![]; |
192 | assert!(walker.pos <= pos); | ||
193 | 70 | ||
194 | while pos > walker.pos && !walker.is_eof() { | 71 | let mut pos = 0; |
195 | walker.forward(); | 72 | let mut cursor = self.start; |
196 | } | 73 | let mut level = 0; |
197 | } | ||
198 | 74 | ||
199 | fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> { | 75 | while pos < n { |
200 | let mut res = vec![]; | 76 | if cursor.eof() { |
201 | let mut walker = self.walker.borrow_mut(); | 77 | break; |
202 | walker.reset(); | 78 | } |
203 | 79 | ||
204 | while walker.pos < n { | 80 | match cursor.token_tree() { |
205 | if let WalkCursor::Token(u, _) = &walker.cursor { | 81 | Some(tt::TokenTree::Leaf(leaf)) => { |
206 | // We only collect the topmost child | 82 | if level == 0 { |
207 | if walker.stack.len() == 0 { | 83 | res.push(leaf.into()); |
208 | if let DelimToken::Token(token) = walker.ts.get(*u) { | ||
209 | res.push(token); | ||
210 | } | 84 | } |
85 | cursor = cursor.bump(); | ||
86 | pos += 1; | ||
211 | } | 87 | } |
212 | // Check whether the second level is a subtree | 88 | Some(tt::TokenTree::Subtree(subtree)) => { |
213 | // if so, collect its parent which is topmost child | 89 | if level == 0 { |
214 | else if walker.stack.len() == 1 { | 90 | res.push(subtree.into()); |
215 | if let DelimToken::Delim(_, is_end) = walker.top().get(*u) { | ||
216 | if !is_end { | ||
217 | let (_, last_idx) = &walker.stack[0]; | ||
218 | if let DelimToken::Token(token) = walker.ts.get(*last_idx) { | ||
219 | res.push(token); | ||
220 | } | ||
221 | } | ||
222 | } | 91 | } |
92 | pos += 1; | ||
93 | level += 1; | ||
94 | cursor = cursor.subtree().unwrap(); | ||
223 | } | 95 | } |
224 | } | ||
225 | 96 | ||
226 | walker.forward(); | 97 | None => { |
98 | if let Some(_) = cursor.end() { | ||
99 | level -= 1; | ||
100 | pos += 1; | ||
101 | cursor = cursor.bump(); | ||
102 | } | ||
103 | } | ||
104 | } | ||
227 | } | 105 | } |
228 | 106 | ||
229 | res | 107 | res |
230 | } | 108 | } |
231 | } | 109 | } |
232 | 110 | ||
233 | impl<'a> Querier for WalkerOwner<'a> { | 111 | impl<'a> Querier for SubtreeWalk<'a> { |
234 | fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool) { | 112 | fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool) { |
235 | self.get(uidx) | 113 | self.get(uidx) |
236 | .map(|tkn| (tkn.kind, tkn.text, tkn.is_joint_to_next)) | 114 | .map(|tkn| (tkn.kind, tkn.text, tkn.is_joint_to_next)) |
@@ -239,22 +117,22 @@ impl<'a> Querier for WalkerOwner<'a> { | |||
239 | } | 117 | } |
240 | 118 | ||
241 | pub(crate) struct SubtreeTokenSource<'a> { | 119 | pub(crate) struct SubtreeTokenSource<'a> { |
242 | walker: WalkerOwner<'a>, | 120 | walker: SubtreeWalk<'a>, |
243 | } | 121 | } |
244 | 122 | ||
245 | impl<'a> SubtreeTokenSource<'a> { | 123 | impl<'a> SubtreeTokenSource<'a> { |
246 | pub fn new<I: Into<TokenSeq<'a>>>(ts: I) -> SubtreeTokenSource<'a> { | 124 | pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> { |
247 | SubtreeTokenSource { walker: WalkerOwner::new(ts) } | 125 | SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) } |
248 | } | 126 | } |
249 | 127 | ||
250 | pub fn querier<'b>(&'a self) -> &'b WalkerOwner<'a> | 128 | pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a> |
251 | where | 129 | where |
252 | 'a: 'b, | 130 | 'a: 'b, |
253 | { | 131 | { |
254 | &self.walker | 132 | &self.walker |
255 | } | 133 | } |
256 | 134 | ||
257 | pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<&tt::TokenTree> { | 135 | pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<tt::TokenTree> { |
258 | let res = self.walker.collect_token_trees(parsed_tokens); | 136 | let res = self.walker.collect_token_trees(parsed_tokens); |
259 | res | 137 | res |
260 | } | 138 | } |