aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe
diff options
context:
space:
mode:
authorbors[bot] <bors[bot]@users.noreply.github.com>2019-05-23 15:31:26 +0100
committerbors[bot] <bors[bot]@users.noreply.github.com>2019-05-23 15:31:26 +0100
commitef00b5af1c7a7a7cac685eff661a10252825d84a (patch)
treeb48cf769d66b2c9366c5488cfa254520730a98a3 /crates/ra_mbe
parenteef24bddc96ddcdbcad5fddb9c0cf0e2ccad7681 (diff)
parent63b67134fdbfedfeab655ac9ff03730575fdf475 (diff)
Merge #1312
1312: Introduce TokenBuffer r=matklad a=edwin0cheng As discussed in Zulip, this PR Introduce `TokenBuffer` , a safe version of `syn` crate `TokenBuffer` which support cursor based traversal of `tt::TokenTree`. This is the basis of incoming refactoring of `TokenSource` iterator based API. This PR do the following things: * Add TokenBuffer in `ra_tt` crate. * Try to use this new API to refactor the `SubtreeSource` to prove it usage. Co-authored-by: Edwin Cheng <[email protected]>
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r--crates/ra_mbe/src/subtree_parser.rs6
-rw-r--r--crates/ra_mbe/src/subtree_source.rs262
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs21
3 files changed, 88 insertions, 201 deletions
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
index f07107414..709b87a38 100644
--- a/crates/ra_mbe/src/subtree_parser.rs
+++ b/crates/ra_mbe/src/subtree_parser.rs
@@ -2,6 +2,7 @@ use crate::subtree_source::SubtreeTokenSource;
2 2
3use ra_parser::{TokenSource, TreeSink}; 3use ra_parser::{TokenSource, TreeSink};
4use ra_syntax::{SyntaxKind}; 4use ra_syntax::{SyntaxKind};
5use tt::buffer::TokenBuffer;
5 6
6struct OffsetTokenSink { 7struct OffsetTokenSink {
7 token_pos: usize, 8 token_pos: usize,
@@ -69,7 +70,8 @@ impl<'a> Parser<'a> {
69 where 70 where
70 F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), 71 F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),
71 { 72 {
72 let mut src = SubtreeTokenSource::new(&self.subtree.token_trees[*self.cur_pos..]); 73 let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
74 let mut src = SubtreeTokenSource::new(&buffer);
73 let mut sink = OffsetTokenSink { token_pos: 0, error: false }; 75 let mut sink = OffsetTokenSink { token_pos: 0, error: false };
74 76
75 f(&src, &mut sink); 77 f(&src, &mut sink);
@@ -85,7 +87,7 @@ impl<'a> Parser<'a> {
85 let res = src.bump_n(parsed_token); 87 let res = src.bump_n(parsed_token);
86 *self.cur_pos += res.len(); 88 *self.cur_pos += res.len();
87 89
88 let res: Vec<_> = res.into_iter().cloned().collect(); 90 let res: Vec<_> = res.into_iter().collect();
89 91
90 match res.len() { 92 match res.len() {
91 0 => None, 93 0 => None,
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index c938acf64..972af4a7c 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -1,50 +1,10 @@
1use ra_parser::{TokenSource}; 1use ra_parser::{TokenSource};
2use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T}; 2use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T};
3use std::cell::{RefCell}; 3use std::cell::{RefCell, Cell};
4use tt::buffer::{TokenBuffer, Cursor};
4 5
5// A Sequece of Token, 6pub(crate) trait Querier {
6#[derive(Debug, Clone, Eq, PartialEq)] 7 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool);
7pub(super) enum TokenSeq<'a> {
8 Subtree(&'a tt::Subtree),
9 Seq(&'a [tt::TokenTree]),
10}
11
12impl<'a> From<&'a tt::Subtree> for TokenSeq<'a> {
13 fn from(s: &'a tt::Subtree) -> TokenSeq<'a> {
14 TokenSeq::Subtree(s)
15 }
16}
17
18impl<'a> From<&'a [tt::TokenTree]> for TokenSeq<'a> {
19 fn from(s: &'a [tt::TokenTree]) -> TokenSeq<'a> {
20 TokenSeq::Seq(s)
21 }
22}
23
24#[derive(Debug)]
25enum DelimToken<'a> {
26 Delim(&'a tt::Delimiter, bool),
27 Token(&'a tt::TokenTree),
28 End,
29}
30
31impl<'a> TokenSeq<'a> {
32 fn get(&self, pos: usize) -> DelimToken<'a> {
33 match self {
34 TokenSeq::Subtree(subtree) => {
35 let len = subtree.token_trees.len() + 2;
36 match pos {
37 p if p >= len => DelimToken::End,
38 p if p == len - 1 => DelimToken::Delim(&subtree.delimiter, true),
39 0 => DelimToken::Delim(&subtree.delimiter, false),
40 p => DelimToken::Token(&subtree.token_trees[p - 1]),
41 }
42 }
43 TokenSeq::Seq(tokens) => {
44 tokens.get(pos).map(DelimToken::Token).unwrap_or(DelimToken::End)
45 }
46 }
47 }
48} 8}
49 9
50#[derive(Debug, Clone, Eq, PartialEq)] 10#[derive(Debug, Clone, Eq, PartialEq)]
@@ -54,183 +14,101 @@ struct TtToken {
54 pub text: SmolStr, 14 pub text: SmolStr,
55} 15}
56 16
57#[derive(Debug, Clone, Eq, PartialEq)]
58enum WalkCursor {
59 Token(usize, TtToken),
60 Eof,
61}
62
63#[derive(Debug)]
64struct SubTreeWalker<'a> {
65 pos: usize,
66 stack: Vec<(TokenSeq<'a>, usize)>,
67 cursor: WalkCursor,
68 ts: TokenSeq<'a>,
69}
70
71impl<'a> SubTreeWalker<'a> {
72 fn new(ts: TokenSeq<'a>) -> SubTreeWalker {
73 let mut res = SubTreeWalker { pos: 0, stack: vec![], cursor: WalkCursor::Eof, ts };
74
75 res.reset();
76 res
77 }
78
79 fn is_eof(&self) -> bool {
80 self.cursor == WalkCursor::Eof
81 }
82
83 fn reset(&mut self) {
84 self.pos = 0;
85 self.stack = vec![];
86
87 self.cursor = match self.ts.get(0) {
88 DelimToken::Token(token) => match token {
89 tt::TokenTree::Subtree(subtree) => {
90 let ts = TokenSeq::from(subtree);
91 self.stack.push((ts, 0));
92 WalkCursor::Token(0, convert_delim(subtree.delimiter, false))
93 }
94 tt::TokenTree::Leaf(leaf) => WalkCursor::Token(0, convert_leaf(leaf)),
95 },
96 DelimToken::Delim(delim, is_end) => {
97 assert!(!is_end);
98 WalkCursor::Token(0, convert_delim(*delim, false))
99 }
100 DelimToken::End => WalkCursor::Eof,
101 }
102 }
103
104 fn current(&self) -> Option<&TtToken> {
105 match &self.cursor {
106 WalkCursor::Token(_, t) => Some(t),
107 WalkCursor::Eof => None,
108 }
109 }
110
111 fn top(&self) -> &TokenSeq {
112 self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts)
113 }
114
115 /// Move cursor forward by 1 step
116 fn forward(&mut self) {
117 if self.is_eof() {
118 return;
119 }
120 self.pos += 1;
121
122 if let WalkCursor::Token(u, _) = self.cursor {
123 self.cursor = self.walk_token(u)
124 }
125 }
126
127 /// Traversal child token
128 fn walk_token(&mut self, pos: usize) -> WalkCursor {
129 let top = self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts);
130 let pos = pos + 1;
131
132 match top.get(pos) {
133 DelimToken::Token(token) => match token {
134 tt::TokenTree::Subtree(subtree) => {
135 let ts = TokenSeq::from(subtree);
136 self.stack.push((ts, pos));
137 WalkCursor::Token(0, convert_delim(subtree.delimiter, false))
138 }
139 tt::TokenTree::Leaf(leaf) => WalkCursor::Token(pos, convert_leaf(leaf)),
140 },
141 DelimToken::Delim(delim, is_end) => {
142 WalkCursor::Token(pos, convert_delim(*delim, is_end))
143 }
144 DelimToken::End => {
145 // it is the top level
146 if let Some((_, last_idx)) = self.stack.pop() {
147 self.walk_token(last_idx)
148 } else {
149 WalkCursor::Eof
150 }
151 }
152 }
153 }
154}
155
156pub(crate) trait Querier {
157 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool);
158}
159
160// A wrapper class for ref cell 17// A wrapper class for ref cell
161#[derive(Debug)] 18#[derive(Debug)]
162pub(crate) struct WalkerOwner<'a> { 19pub(crate) struct SubtreeWalk<'a> {
163 walker: RefCell<SubTreeWalker<'a>>, 20 start: Cursor<'a>,
21 cursor: Cell<Cursor<'a>>,
164 cached: RefCell<Vec<Option<TtToken>>>, 22 cached: RefCell<Vec<Option<TtToken>>>,
165} 23}
166 24
167impl<'a> WalkerOwner<'a> { 25impl<'a> SubtreeWalk<'a> {
168 fn new<I: Into<TokenSeq<'a>>>(ts: I) -> Self { 26 fn new(cursor: Cursor<'a>) -> Self {
169 WalkerOwner { 27 SubtreeWalk {
170 walker: RefCell::new(SubTreeWalker::new(ts.into())), 28 start: cursor,
29 cursor: Cell::new(cursor),
171 cached: RefCell::new(Vec::with_capacity(10)), 30 cached: RefCell::new(Vec::with_capacity(10)),
172 } 31 }
173 } 32 }
174 33
175 fn get<'b>(&self, pos: usize) -> Option<TtToken> { 34 fn get(&self, pos: usize) -> Option<TtToken> {
176 let mut cached = self.cached.borrow_mut(); 35 let mut cached = self.cached.borrow_mut();
177 if pos < cached.len() { 36 if pos < cached.len() {
178 return cached[pos].clone(); 37 return cached[pos].clone();
179 } 38 }
180 39
181 while pos >= cached.len() { 40 while pos >= cached.len() {
182 self.set_pos(cached.len()); 41 let cursor = self.cursor.get();
183 let walker = self.walker.borrow(); 42 if cursor.eof() {
184 cached.push(walker.current().cloned()); 43 cached.push(None);
44 continue;
45 }
46
47 match cursor.token_tree() {
48 Some(tt::TokenTree::Leaf(leaf)) => {
49 cached.push(Some(convert_leaf(&leaf)));
50 self.cursor.set(cursor.bump());
51 }
52 Some(tt::TokenTree::Subtree(subtree)) => {
53 self.cursor.set(cursor.subtree().unwrap());
54 cached.push(Some(convert_delim(subtree.delimiter, false)));
55 }
56 None => {
57 if let Some(subtree) = cursor.end() {
58 cached.push(Some(convert_delim(subtree.delimiter, true)));
59 self.cursor.set(cursor.bump());
60 }
61 }
62 }
185 } 63 }
186 64
187 return cached[pos].clone(); 65 return cached[pos].clone();
188 } 66 }
189 67
190 fn set_pos(&self, pos: usize) { 68 fn collect_token_trees(&mut self, n: usize) -> Vec<tt::TokenTree> {
191 let mut walker = self.walker.borrow_mut(); 69 let mut res = vec![];
192 assert!(walker.pos <= pos);
193 70
194 while pos > walker.pos && !walker.is_eof() { 71 let mut pos = 0;
195 walker.forward(); 72 let mut cursor = self.start;
196 } 73 let mut level = 0;
197 }
198 74
199 fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> { 75 while pos < n {
200 let mut res = vec![]; 76 if cursor.eof() {
201 let mut walker = self.walker.borrow_mut(); 77 break;
202 walker.reset(); 78 }
203 79
204 while walker.pos < n { 80 match cursor.token_tree() {
205 if let WalkCursor::Token(u, _) = &walker.cursor { 81 Some(tt::TokenTree::Leaf(leaf)) => {
206 // We only collect the topmost child 82 if level == 0 {
207 if walker.stack.len() == 0 { 83 res.push(leaf.into());
208 if let DelimToken::Token(token) = walker.ts.get(*u) {
209 res.push(token);
210 } 84 }
85 cursor = cursor.bump();
86 pos += 1;
211 } 87 }
212 // Check whether the second level is a subtree 88 Some(tt::TokenTree::Subtree(subtree)) => {
213 // if so, collect its parent which is topmost child 89 if level == 0 {
214 else if walker.stack.len() == 1 { 90 res.push(subtree.into());
215 if let DelimToken::Delim(_, is_end) = walker.top().get(*u) {
216 if !is_end {
217 let (_, last_idx) = &walker.stack[0];
218 if let DelimToken::Token(token) = walker.ts.get(*last_idx) {
219 res.push(token);
220 }
221 }
222 } 91 }
92 pos += 1;
93 level += 1;
94 cursor = cursor.subtree().unwrap();
223 } 95 }
224 }
225 96
226 walker.forward(); 97 None => {
98 if let Some(_) = cursor.end() {
99 level -= 1;
100 pos += 1;
101 cursor = cursor.bump();
102 }
103 }
104 }
227 } 105 }
228 106
229 res 107 res
230 } 108 }
231} 109}
232 110
233impl<'a> Querier for WalkerOwner<'a> { 111impl<'a> Querier for SubtreeWalk<'a> {
234 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool) { 112 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool) {
235 self.get(uidx) 113 self.get(uidx)
236 .map(|tkn| (tkn.kind, tkn.text, tkn.is_joint_to_next)) 114 .map(|tkn| (tkn.kind, tkn.text, tkn.is_joint_to_next))
@@ -239,22 +117,22 @@ impl<'a> Querier for WalkerOwner<'a> {
239} 117}
240 118
241pub(crate) struct SubtreeTokenSource<'a> { 119pub(crate) struct SubtreeTokenSource<'a> {
242 walker: WalkerOwner<'a>, 120 walker: SubtreeWalk<'a>,
243} 121}
244 122
245impl<'a> SubtreeTokenSource<'a> { 123impl<'a> SubtreeTokenSource<'a> {
246 pub fn new<I: Into<TokenSeq<'a>>>(ts: I) -> SubtreeTokenSource<'a> { 124 pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
247 SubtreeTokenSource { walker: WalkerOwner::new(ts) } 125 SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) }
248 } 126 }
249 127
250 pub fn querier<'b>(&'a self) -> &'b WalkerOwner<'a> 128 pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a>
251 where 129 where
252 'a: 'b, 130 'a: 'b,
253 { 131 {
254 &self.walker 132 &self.walker
255 } 133 }
256 134
257 pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<&tt::TokenTree> { 135 pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<tt::TokenTree> {
258 let res = self.walker.collect_token_trees(parsed_tokens); 136 let res = self.walker.collect_token_trees(parsed_tokens);
259 res 137 res
260 } 138 }
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index d8e344557..0a75305b4 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -47,7 +47,8 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke
47 47
48/// Parses the token tree (result of macro expansion) to an expression 48/// Parses the token tree (result of macro expansion) to an expression
49pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> { 49pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> {
50 let token_source = SubtreeTokenSource::new(tt); 50 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
51 let token_source = SubtreeTokenSource::new(&buffer);
51 let mut tree_sink = TtTreeSink::new(token_source.querier()); 52 let mut tree_sink = TtTreeSink::new(token_source.querier());
52 ra_parser::parse_expr(&token_source, &mut tree_sink); 53 ra_parser::parse_expr(&token_source, &mut tree_sink);
53 if tree_sink.roots.len() != 1 { 54 if tree_sink.roots.len() != 1 {
@@ -62,7 +63,8 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, Expand
62 63
63/// Parses the token tree (result of macro expansion) to a Pattern 64/// Parses the token tree (result of macro expansion) to a Pattern
64pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> { 65pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> {
65 let token_source = SubtreeTokenSource::new(tt); 66 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
67 let token_source = SubtreeTokenSource::new(&buffer);
66 let mut tree_sink = TtTreeSink::new(token_source.querier()); 68 let mut tree_sink = TtTreeSink::new(token_source.querier());
67 ra_parser::parse_pat(&token_source, &mut tree_sink); 69 ra_parser::parse_pat(&token_source, &mut tree_sink);
68 if tree_sink.roots.len() != 1 { 70 if tree_sink.roots.len() != 1 {
@@ -75,7 +77,8 @@ pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandEr
75 77
76/// Parses the token tree (result of macro expansion) to a Type 78/// Parses the token tree (result of macro expansion) to a Type
77pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> { 79pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> {
78 let token_source = SubtreeTokenSource::new(tt); 80 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
81 let token_source = SubtreeTokenSource::new(&buffer);
79 let mut tree_sink = TtTreeSink::new(token_source.querier()); 82 let mut tree_sink = TtTreeSink::new(token_source.querier());
80 ra_parser::parse_ty(&token_source, &mut tree_sink); 83 ra_parser::parse_ty(&token_source, &mut tree_sink);
81 if tree_sink.roots.len() != 1 { 84 if tree_sink.roots.len() != 1 {
@@ -89,7 +92,8 @@ pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, Expan
89pub fn token_tree_to_macro_stmts( 92pub fn token_tree_to_macro_stmts(
90 tt: &tt::Subtree, 93 tt: &tt::Subtree,
91) -> Result<TreeArc<ast::MacroStmts>, ExpandError> { 94) -> Result<TreeArc<ast::MacroStmts>, ExpandError> {
92 let token_source = SubtreeTokenSource::new(tt); 95 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
96 let token_source = SubtreeTokenSource::new(&buffer);
93 let mut tree_sink = TtTreeSink::new(token_source.querier()); 97 let mut tree_sink = TtTreeSink::new(token_source.querier());
94 ra_parser::parse_macro_stmts(&token_source, &mut tree_sink); 98 ra_parser::parse_macro_stmts(&token_source, &mut tree_sink);
95 if tree_sink.roots.len() != 1 { 99 if tree_sink.roots.len() != 1 {
@@ -103,7 +107,8 @@ pub fn token_tree_to_macro_stmts(
103pub fn token_tree_to_macro_items( 107pub fn token_tree_to_macro_items(
104 tt: &tt::Subtree, 108 tt: &tt::Subtree,
105) -> Result<TreeArc<ast::MacroItems>, ExpandError> { 109) -> Result<TreeArc<ast::MacroItems>, ExpandError> {
106 let token_source = SubtreeTokenSource::new(tt); 110 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
111 let token_source = SubtreeTokenSource::new(&buffer);
107 let mut tree_sink = TtTreeSink::new(token_source.querier()); 112 let mut tree_sink = TtTreeSink::new(token_source.querier());
108 ra_parser::parse_macro_items(&token_source, &mut tree_sink); 113 ra_parser::parse_macro_items(&token_source, &mut tree_sink);
109 if tree_sink.roots.len() != 1 { 114 if tree_sink.roots.len() != 1 {
@@ -115,7 +120,8 @@ pub fn token_tree_to_macro_items(
115 120
116/// Parses the token tree (result of macro expansion) as a sequence of items 121/// Parses the token tree (result of macro expansion) as a sequence of items
117pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> { 122pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> {
118 let token_source = SubtreeTokenSource::new(tt); 123 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
124 let token_source = SubtreeTokenSource::new(&buffer);
119 let mut tree_sink = TtTreeSink::new(token_source.querier()); 125 let mut tree_sink = TtTreeSink::new(token_source.querier());
120 ra_parser::parse(&token_source, &mut tree_sink); 126 ra_parser::parse(&token_source, &mut tree_sink);
121 let syntax = tree_sink.inner.finish(); 127 let syntax = tree_sink.inner.finish();
@@ -381,7 +387,8 @@ mod tests {
381 "#, 387 "#,
382 ); 388 );
383 let expansion = expand(&rules, "literals!(foo)"); 389 let expansion = expand(&rules, "literals!(foo)");
384 let tt_src = SubtreeTokenSource::new(&expansion); 390 let buffer = tt::buffer::TokenBuffer::new(&[expansion.clone().into()]);
391 let tt_src = SubtreeTokenSource::new(&buffer);
385 392
386 let query = tt_src.querier(); 393 let query = tt_src.querier();
387 394