aboutsummaryrefslogtreecommitdiff
path: root/crates
diff options
context:
space:
mode:
authorJonas Schievink <[email protected]>2021-06-22 13:23:31 +0100
committerJonas Schievink <[email protected]>2021-06-22 23:21:11 +0100
commitc6669776e1174170c157b685a66026d1a31ede85 (patch)
treed842fcbf296548edee40553157fef124ce9aab74 /crates
parent6504c3c32a9795173fc0c9c94befe7f5d0e7fe9e (diff)
Rewrite `convert_tokens` to use an explicit stack
Diffstat (limited to 'crates')
-rw-r--r--crates/mbe/src/syntax_bridge.rs120
1 files changed, 80 insertions, 40 deletions
diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs
index adf5a56ec..ae6058cbc 100644
--- a/crates/mbe/src/syntax_bridge.rs
+++ b/crates/mbe/src/syntax_bridge.rs
@@ -122,20 +122,25 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
122} 122}
123 123
124fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { 124fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
125 let mut subtree = tt::Subtree { delimiter: None, ..Default::default() }; 125 struct StackEntry {
126 while conv.peek().is_some() { 126 subtree: tt::Subtree,
127 collect_leaf(conv, &mut subtree.token_trees); 127 idx: usize,
128 open_range: TextRange,
128 } 129 }
129 if subtree.token_trees.len() == 1 {
130 if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
131 return first.clone();
132 }
133 }
134 return subtree;
135 130
136 fn collect_leaf<C: TokenConvertor>(conv: &mut C, result: &mut Vec<tt::TokenTree>) { 131 let entry = StackEntry {
132 subtree: tt::Subtree { delimiter: None, ..Default::default() },
133 // never used (delimiter is `None`)
134 idx: !0,
135 open_range: TextRange::empty(TextSize::of('.')),
136 };
137 let mut stack = vec![entry];
138
139 loop {
140 let entry = stack.last_mut().unwrap();
141 let result = &mut entry.subtree.token_trees;
137 let (token, range) = match conv.bump() { 142 let (token, range) = match conv.bump() {
138 None => return, 143 None => break,
139 Some(it) => it, 144 Some(it) => it,
140 }; 145 };
141 146
@@ -144,44 +149,40 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
144 if let Some(tokens) = conv.convert_doc_comment(&token) { 149 if let Some(tokens) = conv.convert_doc_comment(&token) {
145 result.extend(tokens); 150 result.extend(tokens);
146 } 151 }
147 return; 152 continue;
148 } 153 }
149 154
150 result.push(if k.is_punct() && k != UNDERSCORE { 155 result.push(if k.is_punct() && k != UNDERSCORE {
151 assert_eq!(range.len(), TextSize::of('.')); 156 assert_eq!(range.len(), TextSize::of('.'));
157
158 if let Some(delim) = entry.subtree.delimiter {
159 let expected = match delim.kind {
160 tt::DelimiterKind::Parenthesis => T![')'],
161 tt::DelimiterKind::Brace => T!['}'],
162 tt::DelimiterKind::Bracket => T![']'],
163 };
164
165 if k == expected {
166 let entry = stack.pop().unwrap();
167 conv.id_alloc().close_delim(entry.idx, Some(range));
168 stack.last_mut().unwrap().subtree.token_trees.push(entry.subtree.into());
169 continue;
170 }
171 }
172
152 let delim = match k { 173 let delim = match k {
153 T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])), 174 T!['('] => Some(tt::DelimiterKind::Parenthesis),
154 T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])), 175 T!['{'] => Some(tt::DelimiterKind::Brace),
155 T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])), 176 T!['['] => Some(tt::DelimiterKind::Bracket),
156 _ => None, 177 _ => None,
157 }; 178 };
158 179
159 if let Some((kind, closed)) = delim { 180 if let Some(kind) = delim {
160 let mut subtree = tt::Subtree::default(); 181 let mut subtree = tt::Subtree::default();
161 let (id, idx) = conv.id_alloc().open_delim(range); 182 let (id, idx) = conv.id_alloc().open_delim(range);
162 subtree.delimiter = Some(tt::Delimiter { id, kind }); 183 subtree.delimiter = Some(tt::Delimiter { id, kind });
163 184 stack.push(StackEntry { subtree, idx, open_range: range });
164 while conv.peek().map_or(false, |it| it.kind() != closed) { 185 continue;
165 collect_leaf(conv, &mut subtree.token_trees);
166 }
167 let last_range = match conv.bump() {
168 None => {
169 // For error resilience, we insert an char punct for the opening delim here
170 conv.id_alloc().close_delim(idx, None);
171 let leaf: tt::Leaf = tt::Punct {
172 id: conv.id_alloc().alloc(range),
173 char: token.to_char().unwrap(),
174 spacing: tt::Spacing::Alone,
175 }
176 .into();
177 result.push(leaf.into());
178 result.extend(subtree.token_trees);
179 return;
180 }
181 Some(it) => it.1,
182 };
183 conv.id_alloc().close_delim(idx, Some(last_range));
184 subtree.into()
185 } else { 186 } else {
186 let spacing = match conv.peek() { 187 let spacing = match conv.peek() {
187 Some(next) 188 Some(next)
@@ -233,14 +234,44 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
233 id: conv.id_alloc().alloc(r), 234 id: conv.id_alloc().alloc(r),
234 }); 235 });
235 result.push(ident.into()); 236 result.push(ident.into());
236 return; 237 continue;
237 } 238 }
238 _ => return, 239 _ => continue,
239 }; 240 };
240 241
241 leaf.into() 242 leaf.into()
242 }); 243 });
243 } 244 }
245
246 // If we get here, we've consumed all input tokens.
247 // We might have more than one subtree in the stack, if the delimiters are improperly balanced.
248 // Merge them so we're left with one.
249 while stack.len() > 1 {
250 let entry = stack.pop().unwrap();
251 let parent = stack.last_mut().unwrap();
252
253 conv.id_alloc().close_delim(entry.idx, None);
254 let leaf: tt::Leaf = tt::Punct {
255 id: conv.id_alloc().alloc(entry.open_range),
256 char: match entry.subtree.delimiter.unwrap().kind {
257 tt::DelimiterKind::Parenthesis => '(',
258 tt::DelimiterKind::Brace => '{',
259 tt::DelimiterKind::Bracket => '[',
260 },
261 spacing: tt::Spacing::Alone,
262 }
263 .into();
264 parent.subtree.token_trees.push(leaf.into());
265 parent.subtree.token_trees.extend(entry.subtree.token_trees);
266 }
267
268 let subtree = stack.pop().unwrap().subtree;
269 if subtree.token_trees.len() == 1 {
270 if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
271 return first.clone();
272 }
273 }
274 subtree
244} 275}
245 276
246/// Returns the textual content of a doc comment block as a quoted string 277/// Returns the textual content of a doc comment block as a quoted string
@@ -683,6 +714,7 @@ mod tests {
683 algo::{insert_children, InsertPosition}, 714 algo::{insert_children, InsertPosition},
684 ast::AstNode, 715 ast::AstNode,
685 }; 716 };
717 use test_utils::assert_eq_text;
686 718
687 #[test] 719 #[test]
688 fn convert_tt_token_source() { 720 fn convert_tt_token_source() {
@@ -792,4 +824,12 @@ mod tests {
792 let tt = ast_to_token_tree(&struct_def).0; 824 let tt = ast_to_token_tree(&struct_def).0;
793 token_tree_to_syntax_node(&tt, FragmentKind::Item).unwrap(); 825 token_tree_to_syntax_node(&tt, FragmentKind::Item).unwrap();
794 } 826 }
827
828 #[test]
829 fn test_missing_closing_delim() {
830 let source_file = ast::SourceFile::parse("m!(x").tree();
831 let node = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
832 let tt = ast_to_token_tree(&node).0.to_string();
833 assert_eq_text!(&*tt, "( x");
834 }
795} 835}